This commit is contained in:
2023-02-24 16:40:52 +08:00
88 changed files with 7308 additions and 746 deletions
@@ -968,9 +968,6 @@ public class AverageStatisticsController {
*/
public Object baseAction(AverageStatisticsModel model){
Pager pager = model.getPager(25);
if (StringUtils.isBlank(model.getCityDisplay())) {
return model;
}
pager.setTotalRows(avgBaseDao.count(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate()));
model.setBaseList(avgBaseDao.findAll(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate(), pager));
return model;
@@ -987,6 +984,8 @@ public class AverageStatisticsController {
list.add("是否加入指导价");
list.add("调查时间");
list.add("调查人");
list.add("来源");
list.add("来源小区名");
data.add(list);
HSSFWorkbook workbook = new HSSFWorkbook();
HSSFSheet sheet = workbook.createSheet("导入模板");
@@ -994,10 +993,12 @@ public class AverageStatisticsController {
HSSFCell cell0 = row.createCell(0);
HSSFCell cell1 = row.createCell(1);
HSSFCell cell2 = row.createCell(2);
HSSFCell cell3= row.createCell(3);
HSSFCell cell4= row.createCell(4);
HSSFCell cell5= row.createCell(5);
HSSFCell cell6= row.createCell(6);
HSSFCell cell3 = row.createCell(3);
HSSFCell cell4 = row.createCell(4);
HSSFCell cell5 = row.createCell(5);
HSSFCell cell6 = row.createCell(6);
HSSFCell cell7 = row.createCell(7);
HSSFCell cell8 = row.createCell(8);
cell0.setCellValue("城市");
cell1.setCellValue("区域");
cell2.setCellValue("小区名");
@@ -1005,6 +1006,8 @@ public class AverageStatisticsController {
cell4.setCellValue("是否加入指导价");
cell5.setCellValue("调查时间");
cell6.setCellValue("调查人");
cell7.setCellValue("来源");
cell8.setCellValue("来源小区名");
HttpServletResponse response = ActionContext.getContext().getResponse();
response.setCharacterEncoding("UTF-8");
response.setContentType("application/x-download;charset=UTF-8");
@@ -305,11 +305,11 @@ public class EvaluationAuditController {
if(trades!=null && model.getAuditPrice()!=null){
int charge = 0;
if(model.getAuditPrice() > 0 && model.getAuditPrice() <= 100){
charge = 40;
charge = 20;
}else if(model.getAuditPrice() > 100 && model.getAuditPrice() <= 300){
charge = 160;
charge = 70;
}else if(model.getAuditPrice() > 300){
charge = 360;
charge = 80;
}
trades.setCharge((int) charge);
userTradesService.update(trades);
@@ -401,11 +401,11 @@ public class YugudanController {
double price = model.getPeopleEditPrice();
double charge = 0;
if(price > 0 && price <= 100){
charge = 40;
charge = 20;
}else if(price > 100 && price <= 300){
charge = 160;
charge = 70;
}else if(price > 300){
charge = 360;
charge = 80;
}
EvaluationAudit audit = evaluationAuditService.findByEvalutionId(String.valueOf(model.getId()));
@@ -1,6 +1,7 @@
package com.fangjia.bkoff.util;
import com.fangjia.core.Constants;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.HttpRequest;
@@ -8,11 +9,13 @@ import com.fangjia.core.util.HttpRequestException;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.mongo.dao.AvgBaseDao;
import com.fangjia.dal.mongo.model.AverageStatistics;
import com.fangjia.dal.mongo.model.AvgBase;
import com.fangjia.dal.mongo.model.bkoff.GuidingPrice;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.bkoff.GuidingPriceService;
import com.fangjia.service.mongo.fangjia.AverageStatisticsService;
import net.sf.json.JSONObject;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
@@ -47,12 +50,12 @@ import java.util.*;
public class BaseAvgImportServlet extends HttpServlet {
private static final String[] columns = { "城市", "区域", "小区名", "价格", "是否加入指导价", "调查时间", "调查人" };
private static final String[] columns = { "城市", "区域", "小区名", "价格", "是否加入指导价", "调查时间", "调查人" , "来源", "来源小区名"};
private AvgBaseDao avgBaseDao;
private GuidingPriceService guidingPriceService;
private DistrictService districtService;
private AverageStatisticsService averageStatisticsService;
@Override
public void init(ServletConfig config) throws ServletException {
@@ -62,6 +65,7 @@ public class BaseAvgImportServlet extends HttpServlet {
avgBaseDao = (AvgBaseDao) context.getBean("avgBaseDao");
guidingPriceService = (GuidingPriceService) context.getBean("guidingPriceService");
districtService = (DistrictService) context.getBean("districtService");
averageStatisticsService = (AverageStatisticsService) context.getBean("averageStatisticsService");
} catch (Exception e) {
e.printStackTrace();
}
@@ -174,7 +178,7 @@ public class BaseAvgImportServlet extends HttpServlet {
}
private Map<String, Object> checkUploadData(Sheet sheet, String name, long version, Map<String, Object> infoMap,
String realName) {
String realName) {
int totalNum = sheet.getLastRowNum();
System.out.println("上传文件共:" + totalNum + "个");
if (totalNum > 5000) {
@@ -371,7 +375,8 @@ public class BaseAvgImportServlet extends HttpServlet {
} else if (key.equals("价格") && map.get(key) != null) {
String val = formatNumer(map.get(key));
if (StringUtils.isNotBlank(val)) {
int x = Integer.parseInt(val);
double dd = Double.parseDouble(val);
int x = (int) dd;
avgBase.setPrice(x);
}
@@ -385,18 +390,58 @@ public class BaseAvgImportServlet extends HttpServlet {
Date applyDate = (DateUtil.parseDate(map.get(key), "yyyy-MM-dd"));
avgBase.setCreationDate(applyDate == null ? curDate : applyDate);
} else if (key.equals("调查人") && map.get(key) != null) {
avgBase.setUserName(map.get(key));
} else if (key.equals("来源") && map.get(key) !=null){
avgBase.setSource(map.get(key).replaceAll("\\s", ""));
} else if (key.equals("来源小区名") && map.get(key) != null){
avgBase.setSourceCommName(map.get(key).replaceAll("\\s", ""));
}
avgBase.setRegion(RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()));
avgBase.setCategory("district");
}
if (avgBase.getIsGuidancePrice() == 1) {
DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(),
District dis = null;
if(StringUtils.isNotBlank(avgBase.getSourceCommName()) && !avgBase.getSourceCommName().equals(avgBase.getDistrictName())){
DistrictKey districtKey = new DistrictKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()),
avgBase.getDistrictName());
District dis = districtService.findOneByKey(districtKey, "district");
GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), avgBase.getRegion(),
dis = districtService.findOneByKey(districtKey, "district");
if(dis.getAlias() == null){
dis.setAlias(Collections.singletonList(avgBase.getSourceCommName()));
}else if(!dis.getAlias().contains(avgBase.getSourceCommName())){
dis.getAlias().add(avgBase.getSourceCommName());
}
if(dis.getMappingAlias() == null){
dis.setMappingAlias(Collections.singletonList(avgBase.getSourceCommName()));
}else if(!dis.getMappingAlias().contains(avgBase.getSourceCommName())){
dis.getMappingAlias().add(avgBase.getSourceCommName());
}
districtService.save(dis);
}
if(StringUtils.isNotBlank(avgBase.getSource()) && avgBase.getSource().equalsIgnoreCase("GXD")){
AverageStatistics averageStatistics = averageStatisticsService.findSecendAverageStatictics(avgBase.getCity(), avgBase.getRegion(), avgBase.getDistrictName(), "district");
if(averageStatistics != null){
averageStatistics.setGxdEsfPrice(avgBase.getPrice());
}else{
averageStatistics = new AverageStatistics();
averageStatistics.setGxdEsfPrice(avgBase.getPrice());
averageStatistics.setCategory("district");
averageStatistics.setCity(avgBase.getCity());
averageStatistics.setRegion(avgBase.getRegion());
averageStatistics.setName(avgBase.getDistrictName());
if(dis != null){
averageStatistics.setMasterDistrictName(dis.getDistrictName());
averageStatistics.setModificationDate(new Date());
}
}
averageStatisticsService.save(averageStatistics);
}
if (avgBase.getIsGuidancePrice() == 1) {
if(dis == null){
DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(),
avgBase.getDistrictName());
dis = districtService.findOneByKey(districtKey, "district");
}
GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()),
avgBase.getDistrictName(),avgBase.getCreationDate());
if (price != null) {
price.setGuidingPerson(avgBase.getUserName());
@@ -414,7 +459,7 @@ public class BaseAvgImportServlet extends HttpServlet {
price.setGuidingPerson(avgBase.getUserName());
price.setGuidingDate(avgBase.getCreationDate());
price.setGuidingValue(avgBase.getPrice());
price.setEstateType(dis.getEstateType2().indexOf("别墅") > -1 ? "别墅" : "普通住宅");
price.setEstateType(dis.getEstateType2().contains("别墅") ? "别墅" : "普通住宅");
price.setCat("district");
guidingPriceService.save(price);
HttpRequest request = new HttpRequest();
@@ -430,11 +475,10 @@ public class BaseAvgImportServlet extends HttpServlet {
} catch (HttpRequestException e) {
e.printStackTrace();
}
}else{
System.out.println("小区不存在"+ avgBase.getDistrictName());
}
}
if(dis !=null) {
}
}
avgBases.add(avgBase);
@@ -66,6 +66,8 @@
<th style="width:10%;">是否加入指导价</th>
<th style="width:10%;">调查时间</th>
<th style="width:10%;">调查人</th>
<th style="width:10%;">来源</th>
<th style="width:10%;">来源小区名</th>
</tr>
</thead>
<tbody>
@@ -94,7 +96,17 @@
<#if it.userName??>
${it.userName}
</#if>
</td>
</td>
<td align="center">
<#if it.source??>
${it.source}
</#if>
</td>
<td align="center">
<#if it.sourceCommName??>
${it.sourceCommName}
</#if>
</td>
</tr>
</#list>
</#if>
@@ -254,6 +254,9 @@
<#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??>
<td colspan="2"><b>365房产</b></td>
</#if>
<#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??>
<td colspan="2"><b>GXD</b></td>
</#if>
</tr>
<tr>
@@ -345,7 +348,10 @@
<td style="width:70px;color:#0C88C7">新房</td>
<td style="width:70px;color:#D3750A">二手房</td>
</#if>
<#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??>
<td style="width:70px;color:#0C88C7">新房</td>
<td style="width:70px;color:#D3750A">二手房</td>
</#if>
</tr>
<tr>
@@ -430,7 +436,7 @@
<#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??>
<td style ="color:#0C88C7">${it.ganjiNewPrice!}</td>
<td style ="color:#D3750A">${it.ganjiEsfPrice!}</td>
</#if>
</#if>
<#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??>
<td style ="color:#0C88C7">${it.yungujiaNewPrice!}</td>
<td style ="color:#D3750A">${it.yungujiaEsfPrice!}</td>
@@ -443,6 +449,10 @@
<td style ="color:#0C88C7">${it.tsfhouseNewPrice!}</td>
<td style ="color:#D3750A">${it.tsfhouseEsfPrice!}</td>
</#if>
<#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??>
<td style ="color:#0C88C7">${it.gxdNewPrice!}</td>
<td style ="color:#D3750A">${it.gxdEsfPrice!}</td>
</#if>
</tr>
</#list>
@@ -316,11 +316,11 @@
var price = new Number(val)*10000
var charge = 0;
if(price > 0 && price <= 1000000){
charge = 40;
charge = 20;
}else if(price > 1000000 && price <= 3000000){
charge = 160;
charge = 70;
}else if(price > 3000000){
charge = 360
charge = 80;
}
if(price != 0){
$(id).html(charge+'元');
@@ -8,6 +8,6 @@ public interface Constants {
String LOG_MONGO_URI = "mongodb://192.168.0.42:52002";
String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007";
String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020";
String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.19:38080/FangJiaDataService";
String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService";
}
@@ -166,6 +166,8 @@ public class AverageStatistics implements MongoEntity {
private Double tsfhouseEsfDiff;
private Integer tsfhouseNewPrice;
private Double tsfhouseNewDiff;
/**国信达*/
private Integer gxdEsfPrice;
private String level;//价格评级
private Integer lastAvgPrice;//最终价
@@ -1155,4 +1157,13 @@ public class AverageStatistics implements MongoEntity {
public void setTsfhouseNewDiff(Double tsfhouseNewDiff) {
this.tsfhouseNewDiff = tsfhouseNewDiff;
}
@Column(name = "gxd_esf_price")
public Integer getGxdEsfPrice() {
return gxdEsfPrice;
}
public void setGxdEsfPrice(Integer gxdEsfPrice) {
this.gxdEsfPrice = gxdEsfPrice;
}
}
@@ -30,9 +30,9 @@ public class AvgBase implements MongoEntity{
private String category;
private Integer status;
private String dateStr;
private Integer isGuidancePrice;
private String source;
private String sourceCommName;
@Id
@GeneratedValue
@@ -135,4 +135,22 @@ public class AvgBase implements MongoEntity{
public void setDateStr(String dateStr) {
this.dateStr = dateStr;
}
@Column(name = "source")
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
@Column(name = "source_comm_name")
public String getSourceCommName() {
return sourceCommName;
}
public void setSourceCommName(String sourceCommName) {
this.sourceCommName = sourceCommName;
}
}
@@ -24,6 +24,7 @@ import com.fangjia.open.model.*;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
@@ -85,7 +86,6 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
import com.fangjia.service.open.WaitCrawlKeywordsService;
import com.fangjia.service.sql.FjtEvaluationService;
import com.fangjia.service.web.AddressService;
import org.sothis.web.mvc.annotation.Param;
@Authority(role = Role.USER)
public class AddressController {
@@ -116,6 +116,8 @@ public class AddressController {
private static final String GRADE_CONSUME = "消费评分";
public final static OpenConfig CONFIG = OpenConfig.getConfig();
private static final String ES_USER = "developer";
private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3";
/**
* @deprecated see parse2Action
@@ -521,8 +523,65 @@ public class AddressController {
}
MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param);
jsonResult.put("matchResult", matchResult);
if (jsonResult.containsKey("category")) {
if ("office".equals(jsonResult.getString("category"))) {
jsonResult.put("propertyType", "写字楼");
} else if ("shop".equals(jsonResult.getString("category"))) {
jsonResult.put("propertyType", "商铺");
}
}
} else if (jsonResult != null) {
jsonResult.put("matchResult", new MatchDistrictResult());
// 判断是否为自然村
boolean matchedVillage = false;
try {
if (jsonResult.get("searchAddress") != null) {
JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress");
String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null;
String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null;
String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null;
String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null;
if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) {
List<JSONObject> villages = findVillage(city, region, name);
if (CollectionUtils.isNotEmpty(villages)) {
if (villages.size() == 1 || StringUtils.isBlank(township)) {
JSONObject village = villages.get(0).getJSONObject("_source");
jsonResult.put("id", villages.get(0).getString("_id"));
jsonResult.put("province", village.getString("province"));
jsonResult.put("city", village.getString("city"));
jsonResult.put("district", village.getString("region"));
jsonResult.put("township", village.getString("township"));
jsonResult.put("name", village.getString("name"));
jsonResult.put("lng", village.getJSONObject("location").getDouble("lon"));
jsonResult.put("lat", village.getJSONObject("location").getDouble("lat"));
jsonResult.put("propertyType", "自然村");
matchedVillage = true;
} else {
for (JSONObject source : villages) {
JSONObject village = source.getJSONObject("_source");
if (village.getString("township").contains(township) || township.contains(village.getString("township"))) {
jsonResult.put("id", source.getString("_id"));
jsonResult.put("province", village.getString("province"));
jsonResult.put("city", village.getString("city"));
jsonResult.put("district", village.getString("region"));
jsonResult.put("township", village.getString("township"));
jsonResult.put("name", village.getString("name"));
jsonResult.put("lng", village.getJSONObject("location").getDouble("lon"));
jsonResult.put("lat", village.getJSONObject("location").getDouble("lat"));
jsonResult.put("propertyType", "自然村");
matchedVillage = true;
break;
}
}
}
}
}
}
} catch(Exception ex) {
ex.printStackTrace();
}
if (matchedVillage == false) {
jsonResult.put("matchResult", new MatchDistrictResult());
}
}
resultModel.setResult(json.get("result"));
mav.setModel(resultModel);
@@ -918,6 +977,60 @@ public class AddressController {
return ret;
}
private JSONObject addTerm(String name, String value){
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
match.put(name, value);
term.put("term", match);
return term;
}
/***
* 获取自然村
* @param city
* @param region
* @param name
* @return
*/
private List<JSONObject> findVillage(String city, String region, String name) {
JSONObject reqBody = new JSONObject();
reqBody.put("from", 0);
reqBody.put("size", 5);
JSONObject query = new JSONObject();
JSONObject bool = new JSONObject();
JSONArray must = new JSONArray();
must.add(addTerm("city", city));
must.add(addTerm("region", region));
must.add(addTerm("name", name));
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
Base64 token = new Base64();
String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes());
HttpRequest request = new HttpRequest();
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Basic "+authenticationEncoding+"");
try {
String response = request.postWithBody("http://192.168.0.45:9200/village/_search", reqBody, headers, "UTF-8");
JSONObject json = JSONObject.fromObject(response);
if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){
return Collections.emptyList();
}
JSONArray hits = json.getJSONObject("hits").getJSONArray("hits");
List<JSONObject> results = new ArrayList<JSONObject>();
for(int i=0 ;i < hits.size(); i++){
JSONObject jsonObject = hits.getJSONObject(i);
results.add(jsonObject);
}
return results;
} catch (HttpRequestException e) {
e.printStackTrace();
}
return Collections.emptyList();
}
/**
* 加载测试数据
*
@@ -454,9 +454,7 @@ public class WeiXinEvaluationController {
evaluation.setCity(district.getCity());
evaluation.setRegion(district.getRegion());
evaluation.setDistrictName(district.getName());
if(StringUtils.isBlank(evaluation.getAddress())){
evaluation.setAddress(district.getAddress());
}
evaluation.setAddress(district.getAddress());
evaluation.setUserId(userId);
evaluation.setDateCreated(new Date());
evaluation.setLastUpdated(new Date());
@@ -491,8 +489,8 @@ public class WeiXinEvaluationController {
if(fjtUser!=null && fjtUser.getRole()==17) {
System.out.println("渤海地址搜索 "+evaluation.getAddress());
List<JSONObject> bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getAddress());
System.out.println("渤海地址搜索 "+evaluation.getCertificateAddress());
List<JSONObject> bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getCertificateAddress());
JSONObject o = null;
if(bhDists.size()==0){
bhDists = BHBankUtils.getDistrictByBhbank(district.getName());
@@ -545,7 +543,7 @@ public class WeiXinEvaluationController {
if(StringUtils.isBlank(usersDetail.getMobile())){
usersDetail.setMobile(fjtUser.getEmail());
}
System.out.println("渤海询价搜索 "+evaluation.getAddress());
System.out.println("渤海询价搜索 "+evaluation.getCertificateAddress());
JSONObject bhApiResult = BHBankUtils.getInquirePrice(model, bhMatchMap, usersDetail, evaluation.getTotalPrice());
if(bhApiResult!=null && (Integer)bhApiResult.get("errCode")==0){
System.out.println("²³º£Ñ¯¼ÛËÑË÷½á¹û "+bhApiResult.toString());
@@ -1411,11 +1409,16 @@ public class WeiXinEvaluationController {
}
if(uploadfiles.size()>0){
JSONObject jsonObj = BHBankUtils.inquireUploadFiles2(usersDetail.getName(),model.getQueryId(), model.getFileClass(), uploadfiles);
if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) {
mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg")));
if(jsonObj!=null){
if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) {
mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg")));
}else{
mav.setModel(BasicModel.buildServerErrorModel().setMsg(("文件上传成功!")));
}
}else{
mav.setModel(BasicModel.buildServerErrorModel().setMsg(("文件上传成功!")));
mav.setModel(BasicModel.buildServerErrorModel().setMsg("上传失败!"));
}
}
} catch (Exception e) {
e.printStackTrace();
@@ -213,7 +213,7 @@ public class AuthenticationInterceptor implements Interceptor {
if (null == apiUser) {
return ret;
}
if(apiUser.getDeadTime() != null && new Date().after(apiUser.getDeadTime())){
if(apiUser.getDeadTime() != null && DateUtil.truncateDay(new Date()).after(apiUser.getDeadTime())){
return new HttpCode(OpenHttpCodeEnum.USER_EXPIRED);
}
if(!isOldUser(apiUser) && isExpired(token)){
@@ -1,9 +1,7 @@
// JavaScript Document
/* 选择操作 */
// 选择操作-地址
$(document).on('click', '.create-actions00', function () {
/*$(document).on('click', '.create-actions00', function () {
var buttons1 = [
{
text: '请选择地址',
@@ -31,6 +29,7 @@ $(document).on('click', '.create-actions00', function () {
var groups = [buttons1, buttons2];
$.actions(groups);
});
*/
// 选择操作-房型-室
$(document).on('click', '.create-actions01', function () {
@@ -47,13 +47,17 @@
<li>
<div class="item-content">
<div class="item-inner">
<div class="content-block">
<div class="item-title label di">地址:</div>
<input name="fjtEvaluation.address" type="hidden" id="address" placeholder="请选择地址" required>
<p class="di"><a href="#" class="create-actions00">请选择地址</a></p>
<div class="item-title label">产证地址:</div>
<div class="item-input">
<input type="text" placeholder="请输入产证地址" id="certificateAddress"
name="fjtEvaluation.certificateAddress" required>
</div>
<#-- <div class="content-block">-->
<#-- <div class="item-title label di">地址:</div>-->
<#-- <input name="fjtEvaluation.address" type="hidden" id="address" placeholder="请选择地址" required>-->
<#-- <p class="di"><a href="#" class="create-actions00">请选择地址</a></p>-->
<#-- </div>-->
</div>
<div class="item-media"><span>(必填项)</span></div>
</div>
@@ -34,7 +34,7 @@
<div class="topic">
<div class="topicContent topicTop" style="padding:0px 20px;">
<p>${fjtEvaluation.districtName!}</p>
<p style="margin-top:-4px;"><i class="location"></i><small>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("号")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}号</#if>${fjtEvaluation.roomNum!}室</small> <a href="../district/weXinList?cityName=${fjtEvaluation.city!}" class="collectionLink" style="line-height: 32px;">重新评估&nbsp;></a></p>
<p style="margin-top:-4px;"><i class="location"></i><small><#if fjtEvaluation.certificateAddress??>${fjtEvaluation.certificateAddress!}<#else>${fjtEvaluation.address!}</#if> <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("号")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}号</#if>${fjtEvaluation.roomNum!}室</small> <a href="../district/weXinList?cityName=${fjtEvaluation.city!}" class="collectionLink" style="line-height: 32px;">重新评估&nbsp;></a></p>
<p style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;margin-left: 20px;">*请确认与产证地址一致</p>
<span></span>
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0>
+5
View File
@@ -171,6 +171,11 @@
<artifactId>ojdbc14</artifactId>
<version>10.2.0.4.0</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.1</version>
</dependency>
<dependency>
<groupId>com.yahoo.platform.yui</groupId>
<artifactId>yuicompressor</artifactId>
@@ -17,8 +17,8 @@ class AddThirdPartyPrice {
}
def run(){
//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"
def citys = ["林芝"]
//
def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"]
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
@@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
println "txt 生成"
println "txt Éú³É->µ¼Oracle"
saveToPgSql(rets)
}
@@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
return rets;
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -12,6 +12,7 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -28,7 +29,10 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -37,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -71,7 +74,9 @@ class BigdataCommunityBatch extends BaseBatch{
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt Éú³É"
println "txt Éú³É->µ¼Oracle"
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
@@ -169,7 +174,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa101"
@@ -222,4 +227,34 @@ class BigdataCommunityBatch extends BaseBatch{
}
return ret;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@@ -63,7 +66,7 @@ class BigdataDealBatch extends BaseBatch{
curDate = args[1].toString().toInteger()
}
// println "curDate:"+curDate
println "curDate:"+curDate
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
@@ -86,7 +89,6 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
@@ -105,8 +107,10 @@ class BigdataDealBatch extends BaseBatch{
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true);
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -634,7 +638,8 @@ class BigdataDealBatch extends BaseBatch{
def addDealPriceMatch(city){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd")
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
@@ -680,4 +685,35 @@ class BigdataDealBatch extends BaseBatch{
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -57,15 +58,22 @@ class BigdataForsaleBatch extends BaseBatch{
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -74,8 +82,8 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
@@ -102,6 +110,33 @@ class BigdataForsaleBatch extends BaseBatch{
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
allDatas = repeatHouseId(allDatas)
println city + "(新)房源总量:"+allDatas.size()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
}
// def allDatas = createSellDatas(city, retMap.values());
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成->手动导入到Oracle "
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
@@ -116,7 +151,8 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -166,21 +202,38 @@ class BigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
return retMap
}
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
def repeatHouseId(allDatas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(allDatas.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<allDatas.size();i++){
final def houseInfo = allDatas[i];
fixedThreadPool2.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = isExistFromHouse(houseInfo[1])
if(count){
println houseInfo[1]+" 已存在 "+totalCount.get()
}else{
rets << houseInfo
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
latch.await()
return rets
}
@@ -427,8 +480,29 @@ class BigdataForsaleBatch extends BaseBatch{
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
// def cityCode = cityCodeDataMap.get(city)
// def houseIds = getAllHouseIds(commId, cityCode)
// for(def r : rets1){
// if(!houseIds.contains(r.houseId)){
// newRets << r
// }else{
// println r.houseId +" 已存在"
// }
// }
// return newRets
// }
// return [];
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -459,8 +533,11 @@ class BigdataForsaleBatch extends BaseBatch{
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
@@ -586,7 +663,217 @@ class BigdataForsaleBatch extends BaseBatch{
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
def ids = new HashSet()
for(def h : rets1){
ids << h.houseId
}
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(ids && ids.contains(seawater.id)){
continue;
}
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
@@ -599,10 +886,23 @@ class BigdataForsaleBatch extends BaseBatch{
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
@@ -758,12 +1058,6 @@ class BigdataForsaleBatch extends BaseBatch{
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
@@ -786,14 +1080,56 @@ class BigdataForsaleBatch extends BaseBatch{
println ids.size()
}
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def getAllHouseIds(def commid, def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
def distIds = new HashSet()
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
houseIds << it.FORSALEID
});
return distIds;
return houseIds;
}
def isExistFromHouse(def houseId){
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
def count = null
sqlInstance.eachRow(sql,{it->
count = it.count
});
return count;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -16,7 +16,8 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -52,7 +53,7 @@ class CcbCommonUtils {
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
@@ -60,11 +61,13 @@ class CcbCommonUtils {
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
}else{
citys = CcbCommonUtils.citys_92
citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
println "运行城市数量:"+citys.size()
return citys;
@@ -422,7 +425,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","云山尚城")
// q.put("name","鸣翠蓝湾")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -7,13 +7,13 @@ import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"
static def ftpServer = "192.168.0.86"//45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb_evaluation/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
static def ftpPath_rh = "/home/ccb_ronghe/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
@@ -70,16 +70,26 @@ class CcbFtpTestUtils {
}
static def listFileNames(month){
static def listFileNames(type, month){
//ÉÏ´«µ½ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
Vector files = null;
if(type && type=="rh"){
files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel);
}else{
files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
}
return files
}
static def getFileLength(String path, String month){
static def getFileLength(String type, String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
def size = 0;
if(type && type=="rh"){
size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel)
}else{
size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
}
return size
}
@@ -27,6 +27,7 @@ class DealPriceDataProcess {
// task.addLianJiaPrice();
task.deleteThirdPartyData()
// task.delDealByPe()
// task.delDealData()
}
def delDealByPe(){
@@ -370,20 +371,33 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_92
def month = 202209
def citys = CcbCommonUtils.citys_all
def month = 202212
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202210)
def rets = findPeGteThirdParty(city, 202301)
def total = 0
println city +" "+rets.size()
if(rets){
set.add(city)
}
for(def m : rets){
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
println set.toString()
}
def findThirdCount(sqlInstance236, sql){
def count = 0
sqlInstance236.eachRow(sql,{r->
count = r.count()
});
return count
}
def findPeGteThirdParty(cityName, yearmonth){
@@ -398,4 +412,25 @@ class DealPriceDataProcess {
return rets
}
def delDealData(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
if(d['A']){
def count = delDealDataByid(d['A'],d['K'].toString().toDouble(), d['H'].toString().toDouble().toInteger())
println d['B']+" "+count
}
})
}
def delDealDataByid(commid, area, price){
def sql = "select count(*) as count from bigdata_deal_gj where commid='${commid}' and buildarea=${area} and DEALUNITPRICE=${price} " +
"and dealdate>=TO_DATE('2022-11-01','yyyy-MM-dd')"
// sqlInstance236.executeUpdate(sql)
def count = 0
sqlInstance236.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -13,13 +13,15 @@ import org.apache.commons.codec.digest.DigestUtils
import java.sql.ResultSet
//估值 新版 导出所有数据
class ExportOracleAllDataTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
static main(args) {
def task = new ExportOracleAllDataTest()
@@ -41,11 +43,14 @@ class ExportOracleAllDataTest {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
//不会变 10.25
// exportBigdataForsale(new_dists) //2022-07-19
//
// //不会变 10.25
// exportEvaluationRateData()
// exportBigdataPoiData()//
// exportBigdataDistrictData()//
//
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//
@@ -53,29 +58,27 @@ class ExportOracleAllDataTest {
// exportBigdataCommunity()//
// exportCommunityStaging()//没cityid
// exportCommunityBuileType()//
exportThirdPartyPrice2(new_dists)//
// exportBigdataForsale(new_dists) //2022-07-19
//
// exportThirdPartyPrice2(new_dists)//
// exportBigdataDeal( new_dists) //2022-07-01
//
// //第三方网站权重表
////
//// //第三方网站权重表
// exportWebsiteRate();
//
// //计算结果相关
////
//// //计算结果相关
// exportAvmEvaluationGJNB()//
// exportAVMEvaluationItemGJNB()//
//
// exportCityPrice()//
// exportRegionPrice()//
//
// exportCommunityBasePriceGj()//
// exportCommunityBasePrice()//
//
exportCommunityBasePrice()//
// exportCommunityPdescribeGj()//
// exportCommCaseDetailGj()//
def curDate = curMonth.toString()+"28"
// ok文件
// // ok文件
exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
@@ -100,7 +103,6 @@ class ExportOracleAllDataTest {
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
@@ -131,7 +133,6 @@ class ExportOracleAllDataTest {
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
@@ -166,7 +167,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
@@ -194,7 +194,6 @@ class ExportOracleAllDataTest {
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
@@ -228,8 +227,6 @@ class ExportOracleAllDataTest {
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
@@ -267,8 +264,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
@@ -307,8 +302,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
@@ -325,7 +318,7 @@ class ExportOracleAllDataTest {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')";
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd')
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
@@ -344,8 +337,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -390,7 +381,6 @@ class ExportOracleAllDataTest {
}
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
@@ -511,7 +501,6 @@ class ExportOracleAllDataTest {
data << r.SOURCECODE.replaceAll(~'\\r', '')
allRows << data
})
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -528,7 +517,6 @@ class ExportOracleAllDataTest {
def exportThirdPartyPrice2(new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -636,7 +624,6 @@ class ExportOracleAllDataTest {
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
@@ -652,7 +639,6 @@ class ExportOracleAllDataTest {
def exportBigdataForsale(new_dists){
//["北京","上海"]
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
@@ -669,22 +655,24 @@ class ExportOracleAllDataTest {
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
def rets1 = getHouseDatas(sql1, fileName)
if(rets1){
allRows.addAll(rets1)
}
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' ";
allRows = getHouseDatas(sql)
allRows = getHouseDatas(sql, fileName)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
if(allRows && allRows.size()>0){
// index = index + 1
// if(index == 1){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
// }
}
}
@@ -694,7 +682,7 @@ class ExportOracleAllDataTest {
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " +
"from bigdata_forsale_gj where distid in ${newstr}"
def rets2 = getHouseDatas(sql2)
def rets2 = getHouseDatas(sql2, fileName)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
@@ -704,7 +692,7 @@ class ExportOracleAllDataTest {
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def getHouseDatas(sql, fileName){
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
@@ -764,18 +752,23 @@ class ExportOracleAllDataTest {
data << r.SOURCECODE
data << r.ATTACH
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
allRows.clear()
}
});
return allRows
if(allRows.size>0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
allRows.clear()
}
// return allRows
return null
}
def exportAvmEvaluationGJNB(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
@@ -840,7 +833,6 @@ class ExportOracleAllDataTest {
}
def exportAVMEvaluationItemGJNB(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
@@ -894,6 +886,14 @@ class ExportOracleAllDataTest {
def exportCommunityBasePrice(){
def allRows = []
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' ";
@@ -919,18 +919,19 @@ class ExportOracleAllDataTest {
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true);
allRows.clear()
}
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true);
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
CcbFtpTestUtils.ftpUpload(null, fileName1, curDate)
@@ -939,6 +940,11 @@ class ExportOracleAllDataTest {
def exportCommunityBasePriceGj(){
def allRows = []
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' ";
@@ -964,15 +970,17 @@ class ExportOracleAllDataTest {
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
allRows.clear()
}
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
@@ -998,7 +1006,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -1031,9 +1038,6 @@ class ExportOracleAllDataTest {
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -1047,7 +1051,6 @@ class ExportOracleAllDataTest {
def exportCommunityPdescribeGj(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
@@ -1102,11 +1105,19 @@ class ExportOracleAllDataTest {
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
@@ -1114,18 +1125,16 @@ class ExportOracleAllDataTest {
def exportCommCaseDetailGj(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
@@ -1135,15 +1144,16 @@ class ExportOracleAllDataTest {
data << r.PRICE_DATE
data << r.CREATE_DATE
allRows << data
});
if (allRows && allRows.size() > 0) {
index = index + 1
if (index == 1) {
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
} else {
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
}
});
}
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
println "comm_case_detail_gj 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
@@ -1151,7 +1161,6 @@ class ExportOracleAllDataTest {
}
def exportLastFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb_evaluation/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
@@ -1161,7 +1170,6 @@ class ExportOracleAllDataTest {
}
def exportOkFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb_evaluation/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
@@ -1171,15 +1179,14 @@ class ExportOracleAllDataTest {
}
def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpTestUtils.listFileNames(curDate)
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate)
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
def fileName = "/ccb_evaluation/${curDate}.ok"
@@ -43,7 +43,8 @@ class ExportOracleData {
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 ֮ǰ°´publish_dateµ¼³ö
exportBigdataForsale(lastMonthStr +"-18", new_dists) //ÏÖÔÚ°´updatetimeµ¼³ö
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//µÚÈý·½ÍøÕ¾È¨Öرí
@@ -659,13 +660,13 @@ class ExportOracleData {
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
"where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
@@ -12,9 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils
import java.sql.ResultSet
//导出估值 增量数据 新版
class ExportOracleDataTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
@@ -45,7 +46,7 @@ class ExportOracleDataTest {
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataForsale(lastMonthStr +"-18", new_dists) //2022-07-19 按更新时间导出数据
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//第三方网站权重表
@@ -58,7 +59,7 @@ class ExportOracleDataTest {
exportCityPrice(curMonth)//无城市区分
exportRegionPrice(curMonth)//无城市区分
exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分
// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分
exportCommunityBasePrice(curMonth)//无城市区分
exportCommunityPdescribeGj(curMonth)//无城市区分
@@ -658,13 +659,13 @@ class ExportOracleDataTest {
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
"where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
@@ -1149,14 +1150,14 @@ class ExportOracleDataTest {
def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpTestUtils.listFileNames(curDate)
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate)
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
def fileName = "/ccb_evaluation/${curDate}.ok"
@@ -13,6 +13,8 @@ class ExportOracleEvaluationData {
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = null
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
static main(args) {
def task = new ExportOracleEvaluationData()
task.run(args)
@@ -29,7 +31,7 @@ class ExportOracleEvaluationData {
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢
// exportCommunityBasePriceGj(curMonth)//ÒѺÏÏÂÃæµÄºÏ²¢
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
@@ -45,49 +47,53 @@ class ExportOracleEvaluationData {
}
def exportAvmEvaluationGJNB(yearMonth){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth} and CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
@@ -108,38 +114,43 @@ class ExportOracleEvaluationData {
}
def exportAVMEvaluationItemGJNB(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
@@ -158,31 +169,35 @@ class ExportOracleEvaluationData {
}
def exportCommunityBasePrice(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
@@ -201,34 +216,39 @@ class ExportOracleEvaluationData {
}
def exportCommunityBasePriceGj(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -240,21 +260,24 @@ class ExportOracleEvaluationData {
}
def exportCityPrice(month){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
@@ -269,23 +292,27 @@ class ExportOracleEvaluationData {
}
def exportRegionPrice(month){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
@@ -301,68 +328,67 @@ class ExportOracleEvaluationData {
def exportCommunityPdescribeGj(month){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
@@ -370,8 +396,6 @@ class ExportOracleEvaluationData {
def exportCommCaseDetailGj(month){
// def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
@@ -381,8 +405,8 @@ class ExportOracleEvaluationData {
def headers = []
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def cityCode = cityCodeMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month} AND CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -29,7 +29,9 @@ class FjwPricePkCcbPrice {
// task.updatePrice()
// task.addFjwPrice()
// task.addCcbPrice()
task.addMarketPrice()
task.addCcbPriceFromGuided()
// task.addMarketPrice()
}
def addMarketPrice(){
@@ -66,6 +68,24 @@ class FjwPricePkCcbPrice {
})
}
def addCcbPriceFromGuided(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\人工指导价2.xls",{d->
if(d['B'] && !d['O']){
def cityId = d['D']
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
if(map){
d['O'] = map.avg_price
d['P'] = map.calc_type
d['Q'] = map.credit
}
}
}
})
}
def updatePrice(){
def index = 0
@@ -11,6 +11,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -32,6 +33,10 @@ class GuiDingPriceBatch extends BaseBatch{
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
}
@@ -47,7 +52,8 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-23")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -76,10 +82,10 @@ class GuiDingPriceBatch extends BaseBatch{
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
saveToPgSql(rows)
//¸üо²Ì¬Êý¾Ý±í
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -210,5 +216,43 @@ class GuiDingPriceBatch extends BaseBatch{
return distMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_guided_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(300, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
}
}
println "end: "+new Date()
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
pgsqlInstance.executeUpdate(sql)
}
}
}
@@ -0,0 +1,20 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
class PgsqlCommUtils {
public static def database = "warehouse"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.gujia_comm_id
});
return set
}
}
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
/**
@@ -18,6 +19,9 @@ import org.bson.types.ObjectId
@BatchOwner(Watcher.GaoJuan)
class StagingDistrictBatch extends BaseBatch{
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new StagingDistrictBatch(), args);
}
@@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{
//ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
saveToPgSql(rows)
}
def createCityData(def dists, def fenQiMap){
@@ -86,4 +92,34 @@ class StagingDistrictBatch extends BaseBatch{
}
return fenQiMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -45,6 +45,139 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d->
// if(d['J']){
// def cityId = cityCodeDataMap.get(d['H'])
// def map = findCcbPriceCredit(sqlInstance236, cityId, d['J'])
// println d['H'] +" "+d['J']+ map.toString()
// if(map){
// d['Q'] = map.credit
// d['R'] = map.month
// d['U'] = map.price
// }
//// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J'])
//// if(thirdMap){
//// d['S'] = thirdMap.price
//// d['T'] = thirdMap.month
//// }
// }
//
//})
def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.year_month
map.credit = r.CONFIDENCE
map.price = r.avg_price
});
return map
}
def findThirdPriceBySource(sqlInstance, cityid, commid){
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" +
"order by price_date desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.price_date
map.price = r.community_price
});
return map
}
//for(def c : citys){
// def cityCode = cityCodeDataMap.get(c)
// println c +" "+cityCode
// def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301)
////
// checkNoPriceDist(sqlInstance236, priceDists, cityCode)
//}
def checkNoPriceDist(sqlInstance236,priceDists, cityid){
def commids = []
def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
for(def commid : commids){
if(!priceDists.contains(commid)){
updateDistStatus(sqlInstance236, commid)
}
}
return commids
}
def updateDistStatus(sqlInstance236, commid){
def sql = "update bigdata_community_gj set status=1 where commid='${commid}'";
sqlInstance236.executeUpdate(sql)
}
def findAllPriceDist(sqlInstance236, cityid, yearMonth){
def commids = []
def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} ";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
return commids
}
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
//
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def rets = []
//for(def c : citys){
// def citycode = cityCodeDataMap.get(c)
// def datas = findAllData(sqlInstance236, citycode)
// def totalCount = datas.size()
// def landareaCount = 0;
// def buildareaCount = 0
// def greeningrateCount = 0
// def plotratioCount = 0
// for(def m : datas){
// if(m.landarea){
// landareaCount ++
// }
// if(m.buildarea){
// buildareaCount++
// }
// if(m.greeningrate){
// greeningrateCount++
// }
// if(m.plotratio){
// plotratioCount++
// }
// }
//
// def landAreaPre = landareaCount*100/totalCount
// def buildAreaPre = buildareaCount*100/totalCount
// def greeningratePre = greeningrateCount*100/totalCount
// def plotratioPre = plotratioCount*100/totalCount
// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
//}
//
//ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
def findAllData(sqlInstance236, citycode){
def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'"
def rets = []
sqlInstance236.eachRow(sql,{r->
rets << ["landarea":r.landarea, "buildarea":r.buildarea, "greeningrate":r.greeningrate, "plotratio":r.plotratio]
});
return rets
}
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
////println s
@@ -447,23 +580,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
////},0)
//
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrictById(d['D'])
if(dbo){
def udata = new BasicDBObject()
// udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println dbo._id.toString() +" 没找到"
}
}
},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
// if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['D'])
// if(dbo){
// def udata = new BasicDBObject()
//// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dbo._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println dbo._id.toString() +" 没找到"
// }
// }
//},0)
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
@@ -1037,6 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){
//}
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(1).xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['G'] = dist.city
d['H'] = dist.region
d['I'] = dist.name
}
})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx"
//def datas = ExcelUtils.read(fileName, 0, 1)
@@ -1099,6 +1241,30 @@ def findThirdCommCount(cityid, sqlInstance){
//ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d->
// def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district")
// if(obj){
// def count = findGuidedPrice(sqlInstance236, obj._id.toString())
// if(count){
// d['M'] = "存在"
// }
// d['N'] = obj.name_cat
// d['O'] = obj.estate_type2
// }
//
//})
def findGuidedPrice(sqlInstance, id){
def count = 0
def sql = "select count(*) as count from guided_price_gj where commid='${id}'"
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202210//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202208
def endDate = 202210
def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202211
def endDate = 202301
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -36,6 +36,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
}
@@ -45,7 +49,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
@@ -67,7 +71,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
saveToPgSql(rows)
}
@@ -618,4 +624,32 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_third_party_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
}
}
println "end: "+new Date()
}
}
@@ -50,8 +50,25 @@ class TradeStatLog {
// task.deleteDealData2()
// task.statTableCount()
// task.filterTrade()
// task.updateDealPriceType()
}
def updateDealPriceType(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it->
if(it.total_price instanceof Double){
index ++
println index+" "+it.city+" "+it.total_price+" "+it._id.toString()
def udata = new BasicDBObject()
udata.put("total_price", (int)it.total_price)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata))
}
}
}
}
def statTableCount(){
@@ -73,15 +90,15 @@ class TradeStatLog {
}
def run(){
def citys = CcbCommonUtils.citys_92;
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","11月总量","12月总量","23年1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-08-01"
def date = "2022-09-01"
def endDate = "2022-10-01"
def newDate = "2022-11-01"
def startDate = "2022-11-01"
def date = "2022-12-01"
def endDate = "2023-01-01"
def newDate = "2023-02-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -0,0 +1,115 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class TestData {
static main(args) {
def task = new TestData()
// task.run()
task.update()
}
def update(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ1.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
if(d['E'] && d['G']=="È˹¤"){
def q = new BasicDBObject()
def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
q.put("lat2", dLat2)
q.put("lng2", dLng2)
Long lng = d['E'].toString().toDouble() * 10000000000L
Long lat = d['F'].toString().toDouble() * 10000000000L
q.put("lng", lng)
q.put("lat", lat)
DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q))
}
}
});
}
def run(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def loc = getLocation(dist.city, dist.region, dist.name)
if(loc){
d['E'] = loc.x
d['F'] = loc.y
println d['C']+" "+loc.x+" "+loc.y
}
}
// if(!d['E']){
// def loc = getAmapLocation("ÁúÑÒÊÐ", d['B'], d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
// if(!d['E']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def loc = getAmapOldLocation(dist.city, dist.region, d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
//
// }
});
}
def getLocation(city, region, name){
String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
loc = data.location
}
})
return loc;
}
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
def getAmapOldLocation(city, region, name){
String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
}
@@ -0,0 +1,622 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
class ExportPgsqlBaseEvaluation {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
def citys = null
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def database = "base"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlBaseEvaluation()
task.run(args)
}
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.reverse()
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//
// //第三方网站权重表
// exportWebsiteRate();
//
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
// exportCommunityStaging()//没cityid
// exportBigdataCommunity()//
//
// exportCommunityBuildType()//
//
// exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(DateUtils.formatDateByPattern(new Date(), "yyyy-MM") +"-18", new_dists) //现在按updatetime导出
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
}
def exportEvaluationRateData() {
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_ceft_id
data << r.gujia_ceft_tp
data << r.gujia_bds_ceft
data << r.gujia_bds_seq
data << r.gujia_urbn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false)
}
def exportBigdataPoiData() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_fsid
data << r.poi_nm
data << r.gujia_tp
data << r.gujia_tp_ecd
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false)
}
def exportBigdataDistrictData() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_admndiv_nm_cpa
data << r.urbn_id
data << r.gujia_natstd_cd
data << r.gujia_dspl_seq
data << r.gujia_admndiv_cgy
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_lgc_admndiv_dsc
data << r.gujia_lgc_admndiv_cd
data << r.gujia_adiv_cd_val
data << r.gujia_bds_data_tm_val
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false)
}
def exportWebsiteRate(){
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujiadatawebst_src_id
data << r.gujia_data_webst_src
data << r.gujiadatawebstmpng_id
data << r.gujia_bds_wgt
data << r.gujia_bds_tp
data << r.gujia_src_co
data << r.urbn_id
data << r.gujia_urbn
data << r.data_crt_tm
data << r.data_udt_tm
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false)
}
def exportGuidedPrice() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_dt
data << r.gujia_bds_drc_prc
data << r.gujia_add_psn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false)
}
def exportCommunityGuidedPrice(def date) {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_drc_prc
data << r.gujia_bds_drc_prc_src
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false)
}
def exportBigdataCommunity(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100)
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false)
}
def exportCommunityStaging() {
def allRows = []
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_instm_id
data << r.gujia_comm_id
data << r.gujiacomm_prim_prd_id
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false)
}
def exportCommunityBuildType() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_tp_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_bds_data_tm_val
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false)
}
def exportThirdPartyPrice2(price_date, new_dists) {
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujiabds_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujiabds_avprc_dt_val='${price_date}' ";
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
}
def getThirdPartyDatas(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_dtsr_id
data << r.gujia_dtsr_dsc
data << r.gujiabds_avprc_dt_val
data << r.gujia_bds_comm_avprc
data << r.gujiabdscomm_hgst_prc
data << r.gujiabdscomm_lwst_prc
data << r.data_udt_tm
allRows << data
})
return allRows;
}
def exportBigdataDeal(def dealDate, def new_dists) {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false)
}
def dealData(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bds_mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_bds_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujiabdstot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_bds_trdtckt_prc
data << r.gujia_bds_mdl_tot_prc
data << r.gujia_bds_dcl_tot_prc
data << r.gujiabds_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_bds_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiabdslndfgrngvarea
data << r.gujiabdslutbl_gv_area
data << r.gujiabdssuntblgv_area
data << r.gujiabdsbayindowgarea
data << r.gujiabdstowerfrgvarea
data << r.gujia_bds_grg_gv_area
data << r.gujiabdsundgdrmgvarea
data << r.gujiabds_othr_gv_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_bds_mkdam
data << r.gujia_bds_data_tm_val
allRows << data
});
return allRows
}
def exportBigdataForsale(publishDate, new_dists){
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
println city +" "+allRows.size()
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true);
}
}
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
pgsqlInstance.eachRow(sql,{r->
index = index+1
def data = []
data << r.urbn_id
data << r.gujia_sell_id
data << r.gujia_listg_hs_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_bds_unitprc
data << r.gujia_bds_tot_prc
data << r.gujia_sell_pymtmod
data << r.gujia_bds_wbt_floor
data << r.gujiabdstot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_indr_stc_val
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujiawthrext_lift_ind
data << r.gujiajiladderjiactval
data << r.anc_tm
data << r.exdat
data << r.gujia_wthr_strt
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.gujia_anc_psn
data << r.gujia_anc_psn_tel
data << r.gujia_dsc_inf_cntnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_apd_inf
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println tableName+" 上传Ftp完成......"
}
}
@@ -0,0 +1,443 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import org.apache.commons.codec.digest.DigestUtils
class ExportPgsqlEvaluationResult {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
def citys = null
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def database = "t_app_p9pha_plda."
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlEvaluationResult()
task.run(args)
}
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
// exportCommunityBasePrice(curMonth)
exportCommunityBasePrice1(curMonth)
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
// exportInOk()
//
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
def exportAvmEvaluationGJNB(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_bds_yrmo = ${yearMonth} and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_bds_yrmo
data << r.gujia_bds_comm_num
data << r.gujiabdsvalmnycommnum
data << r.gujia_prc_cvr_rate
data << r.gujiabdsmnldpccommnum
data << r.gujia_mnl_drc_prc_pct
data << r.gujiabdssamp_comm_num
data << r.gujiaext_samp_clc_pct
data << r.gujiabdsnervetncomnum
data << r.gujianervethrunclcpct
data << r.gujiabds_tst_case_num
data << r.gujiabds_tst_comm_num
data << r.gujiatst_comm_num_pct
data << r.gujia_ppe15_cvr_rate
data << r.gujia_rto_mdn
data << r.gujia_rto_lv_san_ceft
data << r.gujiabdsmdltstcasenum
data << r.gujiabdsmdltstcommnum
data << r.gujiamdltstcommnumpct
data << r.gujiamdltst_ppe15_pct
data << r.gujiamdlrtolvsan_ceft
data << r.gujia_mdl_md_bit_rto
data << r.gujiabdstrdpttcasenum
data << r.gujiabdstrdpttcommnum
data << r.gujiatrdpttcommnumpct
data << r.gujiatrdpttstppe15pct
data << r.gujiatrdptrolvsanceft
data << r.gujiatrdpt_md_bit_rto
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false)
createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false)
}
def exportAVMEvaluationItemGJNB(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val = ${yearMonth} AND urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_rgon_id
data << r.gujia_rgon_nm
data << r.gujia_comm_id
data << r.gujia_comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_avprc
data << r.gujia_bds_clc_tp
data << r.gujia_bds_rlb
data << r.gujiabds_to_mkprc_mny
data << r.mdl_dt
data << r.gujiabds_valt_unitprc
data << r.gujia_area
data << r.gujia_dcrt_val
data << r.gujia_twrd_val
data << r.gujia_bds_floor_num
data << r.gujiabdstot_floor_num
data << r.gujia_data_tp
data << r.gujia_rto
data << r.gujia_mns_rto
data << r.data_crt_tm
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false)
createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false)
}
def exportCityPrice(month){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${month} AND
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false)
}
def exportRegionPrice(month){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; //gujiabds_clc_yrmo_val=${month} AND
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gxda_admndiv_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false)
}
def exportCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false)
createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false)
}
def exportCommunityBasePrice1(yearMonth){
def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true);
allRows.clear()
}
});
}
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true);
}
}
def exportCommunityPdescribeGj(month){
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where gujia_bds_mo_val=${month} AND urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_comm_id
data << r.urbn_id
data << r.urbn_nm
data << r.admndiv_ecd
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_als_dsc
data << r.gujia_bds_bss_prc
data << r.gujiabds_cfdc_dgr_grd
data << r.gujiabdsrt3mlgcasenum
data << r.gujiabdslistgclwstprc
data << r.gujiabdslistgcaemdval
data << r.gujiabdslistgcase_avg
data << r.gujiabdslistgcemaxprc
data << r.gujia_bds_mo_val
data << r.gujiabdsavgwminvalavg
data << r.gujiabdsavgwmaxvalavg
data << r.gujia_bds_rel_err_val
data << r.gujia_dvtdgr_dsc
data << r.gujia_als_prc_rng
data << r.gujiaprcminandmax_rng
data << r.gujia_dsc
data << r.gujiabds_lmth_bss_prc
data << r.gujiabds_wthr_mnl_val
data << r.gujiabds_lmth_fnl_prc
data << r.data_crt_tm
data << r.gujiabdsnervetuntwprc
data << r.gujia_prc_src
data << r.gujiabdsabtjlcmbitval
data << r.gujiabdsabtjlgcaseavg
data << r.gujia_indx_lbl
data << r.gujia_prim_pty_tp
data << r.gujia_comm_tp_val
data << r.gujiabds_ltst_mdl_prc
data << r.gujia_ltst_mdl_dt
data << r.gujiabdsidxnmo_moonmo
data << r.gujiabdsa1yiminmmonmo
data << r.gujiabdsidxnyr_moonmo
data << r.gujiabdsat1yspdgrscor
data << r.gujiabdsarnd1kscavprc
allRows << data
});
}
createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false)
}
def exportCommCaseDetailGj(month){
def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}";
def allRows = []
pgsqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_case_id
data << r.gujia_bds_crterm
data << r.data_crt_tm
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
// allRows.clear()
// }
});
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
}
def exportMd5(){
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = linux_prefix + "md5.txt"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "md5.txt"
}
createFileToFtp("md5.txt",fileRows, fileName, curDate, false)
}
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
def fileName = linux_prefix + "/${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println "ok文件上传Ftp完成......"
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
//CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
//println tableName+" 上传Ftp完成......"
}
}
@@ -0,0 +1,618 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
class ExportPgsqlRonghe {
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
"postgres", "postgres", "org.postgresql.Driver")
def citys = []
def database = "base."
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def linux_prefix = "/ccb_ronghe/${curDate}/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlRonghe()
task.run(args)
}
def run(args) {
def new_dists = []
citys = CcbCommonUtils.getRunningCitys(args)
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists)
if (curDate.toString().endsWith("10")) {
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportInOk()
// }
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
}
}
def exportBigdataDealFJW2(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
}
createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false)
}
def dealData(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_id
data << r.mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujia_tot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.gujiaconstructionarea
data << r.gujia_set_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_room
data << r.gujia_dept
data << r.gujia_ktch_num
data << r.gujia_toliet
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_trdtckt_prc
data << r.gujia_mdl_tot_prc
data << r.gujia_dcl_tot_prc
data << r.gujia_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiagvlndfargrdnarea
data << r.gujia_gv_lu_tbl_area
data << r.gujia_gv_sun_tbl_area
data << r.gujiagvbaywindow_area
data << r.gujiagvarea_tower_flr
data << r.gujia_gv_grg_area
data << r.gujiagvundgdroom_area
data << r.gujia_gv_othr_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_mkdam
data << r.gujia_data_tm
allRows << data
});
return allRows
}
def exportEvaluateRate(){
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_ceft_id
data << r.gujia_ceft_tp
data << r.gujia_ceft
data << r.gujia_seq
data << r.gujia_urbn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildprojFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false)
}
def exportBigdataCommlayoutpicFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false)
}
def exportBigdataCommunityGJRH(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_comm_fmnm
data << r.gujia_admndiv_id
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_dtl_adr
data << r.gujiaocpoflandtotarea
data << r.gujia_bld_tot_area
data << r.gujia_comm_tp_val
data << r.gujiawthr_new_pan_ind
data << r.gujia_grnrate
data << r.gujia_vol_rate
data << r.gujia_park_num
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_stc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommfeeGj(){
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.co_id
data << r.gujia_eps_val
data << r.gujia_hs_use
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_comm_eps_tp
data << r.gujia_eps_cmnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhousetypeGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_comm_hs_tp
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildyearGj(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhouseuseGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_hs_use
data << r.gujia_ladder_accnum
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataPositionGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_obj_tp
data << r.gujia_obj_id
data << r.gujia_lo_tp
data << r.gujia_sn
data << r.lgt
data << r.ltt
data << r.gujia_hignas_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false)
}
def exportBigdataDistrictFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_admndiv_nm_cpa
data << r.urbn_id
data << r.gujia_natstd_cd
data << r.gujia_dspl_seq
data << r.gujia_admndiv_cgy
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false)
}
def exportBigdataAliasFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.rel_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_pcsst
data << r.gujia_vld_dt
data << r.gujia_vrsn_dsc
data << r.gujia_wthr_prim_nm
data << r.gujia_dtsrc
data << r.gujia_dsc
data << r.gujia_als_lvl
data << r.gujia_adr_tp_ecd
data << r.gujia_wthr_adt_ind
data << r.gujia_wthr_dspl_ind
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildingFJW(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_bld_als_dsc
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.floor_num
data << r.gujia_unit_num
data << r.gujia_accnum_val
data << r.gujiawthrext_lift_ind
data << r.gujia_lift_num
data << r.gujiajiladderjiactval
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_vld_ind
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false)
}
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
}
def fileName = linux_prefix+"${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println tableName+" 上传Ftp完成......"
}
}
@@ -27,6 +27,8 @@ class BigdataCitypriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
}
@@ -238,7 +240,7 @@ class BigdataCitypriceBatch extends BaseBatch{
if (sell_avg_price || rent_avg_price) {
def row = []
row << cityCodeDataMap.get(dist.city)
row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
row << dist._id.toString()
if (sell_avg_price) {
@@ -362,7 +364,7 @@ class BigdataCitypriceBatch extends BaseBatch{
if (sell_avg_price || rent_avg_price) {
def row = []
row << cityCodeDataMap.get(city)
row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
if(region){
row << regionCodeDataMap.district_code
@@ -31,8 +31,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = null
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def batchsize = 1000
static main(args) {
@@ -42,13 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
if(citys.size()==92){
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else{
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
@@ -4,10 +4,12 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import java.sql.ResultSet
@@ -27,11 +29,16 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
@@ -49,7 +56,28 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
exportThirdPartyPrice2(thirdPartyMonth, new_dists)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
exportCommunityBasePrice(lastMonth)
exportCommunityPdescribeGj(lastMonth)
exportGuideStatistics(lastMonth)
exportBigdataSupplierStatistics(lastMonth)
exportBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -526,7 +554,7 @@ class ExportDistrictRongHe extends BaseBatch{
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("28")){
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
@@ -801,6 +829,75 @@ class ExportDistrictRongHe extends BaseBatch{
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportThirdPartyPrice2(price_date, new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and distid in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = null
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} ";
}else{
sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}'";
}
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload('rh', fileName, curDate)
}
def getThirdPartyDatas(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.COMMNAME
data << r.DATA_SOURCE_ID
data << r.DATA_SOURCE
data << r.PRICE_DATE
data << r.COMMUNITY_PRICE
data << r.COMMUNITY_MAX_PRICE
data << r.COMMUNITY_MIN_PRICE
data << r.UPDATETIME
allRows << data
})
return allRows;
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
@@ -812,6 +909,393 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'";
}else{
sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'";
}else{
sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where CITYID='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
}else{
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.MODEL_TYPE
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMM_SUM_COUNT
data << r.COMM_COUNT
data << r.RATIO
data << r.TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "guide_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "guide_statistics 上传Ftp完成......"
}
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}' and year_month=${time}";
}else{
sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.DATA_SOURCE
data << r.THIRD_COMM_COUNT
data << r.MODEL_TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_supplier_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}' and year_month=${time}";
}else{
sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITY_CODE
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.ESTIMATE_DEAL_COUNT
data << r.ESTIMATE_COMM_COUNT
data << r.DEAL_COUNT
data << r.DEAL_COMM_COUNT
data << r.SALE_COUNT
data << r.SALE_COMM_COUNT
data << r.MODEL_TYPE
data << r.TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_modeling_base 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_CITYPRICE 上传Ftp完成......"
}
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.DISTID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DISTPRICE 上传Ftp完成......"
}
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.COMMID
data << r.SALEPRICE?r.SALEPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
data << r.RENTPRICE?r.RENTPRICE:''
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.HOUSEUSE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.DATATYPE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMPRICE 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
@@ -823,7 +1307,132 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
if(city=="苏州"){
continue
}
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " +
"from bigdata_forsale_gj where distid in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
}
println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println r.CITYNAME +" "+index
def data = []
data << r.CITYID
data << r.FORSALEID
data << r.PROPCODE
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.UNITPRICE
data << r.TOTALPRICE
data << r.PAYTYPE
data << r.FLOORNUM
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.HOUSESTRU
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEASPECT
data << r.FITMENT
data << r.IFELEVATOR
data << r.ELEVATORHOUSE
data << r.PUBLISHTIME
data << r.EXPIRETIME
data << r.IFFIRST
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.PUBLISHER
data << r.PHONE
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
// data << r.ATTACH //融合房源没有这个字段
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
@@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
@@ -25,40 +26,43 @@ class ExportDistrictRongHeTest extends BaseBatch{
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
// if(curDate.toString().endsWith("10")){
// exportBigdataBuildprojFjw()
// exportBigdataCommlayoutpicFjw()
// exportEvaluateRate()
//
// exportBigdataCommunityGJRH()//
// exportBigdataBuildingFJW()//
// exportBigdataCommfeeGj()//
//
// exportBigdataCommbuildstruGJ()//
// exportBigdataCommhousetypeGJ()//
// exportBigdataCommbuildyearGj()//
// exportBigdataCommhouseuseGJ()//
// exportBigdataPositionGJ()//
// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()//
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
exportInOk()
// }
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
}
@@ -66,9 +70,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys){
@@ -133,9 +137,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -162,9 +166,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -190,9 +194,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -228,9 +232,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -258,9 +262,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -291,9 +295,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -323,9 +327,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -358,9 +362,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -382,9 +386,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
@@ -468,7 +472,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
CcbFtpTestUtils.ftpUpload("ccb_ronghe", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
@@ -482,9 +486,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
@@ -511,8 +515,13 @@ class ExportDistrictRongHeTest extends BaseBatch{
// }
// }
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ ";//
allRows.addAll(createDeal(sql))
for(def city : citys) {
if(CcbCommonUtils.citys_252.contains(city)){
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}'";
allRows.addAll(createDeal(sql))
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -675,9 +684,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
@@ -686,9 +695,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
@@ -743,14 +752,16 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -789,14 +800,14 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";//
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -836,14 +847,14 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
@@ -899,9 +910,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -910,9 +921,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -921,23 +932,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "融合统计文件长度ok文件 上传Ftp完成......"
}
}
@@ -16,7 +16,7 @@ class SSHHelperTest {
private Session session = null;
private Channel channel = null;
private String sftpHost = "192.168.0.86";
private String sftpHost = "192.168.0.45";
private int sftpPort = 17495;
private String sftpUserName = "ccb";
private String sftpPassword = "ccb!@#";
@@ -0,0 +1,82 @@
package com.fangjia.script.chongqing
import com.sun.org.apache.xml.internal.security.utils.Base64;
import org.apache.commons.lang3.StringUtils;
import javax.crypto.Cipher;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.DESedeKeySpec;
import javax.crypto.spec.IvParameterSpec;
import java.security.Key;
class Des3 {
private final static String encoding = "UTF-8";
static main(args) {
def task = new Des3()
task.run()
}
def run(){
try {
String key = "111036369260679051122113";
String iv = "23235645";//IV length must be 8 bytes long
//加密
String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv);
System.out.println("密文: " + encryptStr);
//解密
String decryptStr = decode(encryptStr, key, iv);
System.out.println("明文: " + decryptStr);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 方法描述:3DES加密
* @author hyang
*
* @param plainText 明文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 密文
* @throws Exception
*/
public static String encode(String plainText, String secretKey, String iv)
throws Exception {
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding");
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.ENCRYPT_MODE, deskey, ips);
byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding));
return Base64.encode(encryptData);
}
/**
* 方法描述: 3DES解密
* @author hyang
*
* @param encryptText 密文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 明文
* @throws Exception
*/
public static String decode(String encryptText, String secretKey, String iv)
throws Exception {
if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) {
return "";
}
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.DECRYPT_MODE, deskey, ips);
byte[] decryptData = cipher.doFinal(Base64.decode(encryptText));
return new String(decryptData, encoding).trim();
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.chongqing
import com.fangjia.SecretUtils
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.spec.SecretKeySpec
class Des3Test {
//定义加密算法,有DES、DESede(即3DES)、Blowfish
private static final String Algorithm = "DESede";
private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113";
/**
* 加密方法
* @param src 源数据的字节数组
* @return
*/
public static byte[] encryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥
Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类
c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/**
* 解密函数
* @param src 密文的字节数组
* @return
*/
public static byte[] decryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm);
Cipher c1 = Cipher.getInstance(Algorithm);
c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/*
* 根据字符串生成密钥字节数组
* @param keyStr 密钥字符串
* @return
* @throws UnsupportedEncodingException
*/
public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{
byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0
byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组
/*
* 执行数组拷贝
* System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位)
*/
if(key.length > temp.length){
//如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, temp.length);
}else{
//如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, key.length);
}
return key;
}
/**
* @param args
*/
public static void main(String[] args) {
String msg = "3DES加密解密案例";
System.out.println("【加密前】:" + msg);
//加密
// byte[] secretArr = Des3Test.encryptMode(msg.getBytes());
// System.out.println("【加密后】:" + new String(secretArr));
String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"
//解密
byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8"));
System.out.println("【解密后】:" + new String(myMsgArr));
}
}
@@ -0,0 +1,703 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class HailanTest {
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\海南\\统计.xlsx"
// match(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
hailanStat(inputfile)
}
def addHouse(inputfile){
def datas = []
datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向",
"景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"]
def index = 0
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['L']){
index ++
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
println index +" "+ row['C']+" "+ rets.size()
if(rets){
def lastRets = []
for(def r : rets){
def fjwPrice = row['M']
if(fjwPrice){
def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price)
if(precent>=10){
continue;
}
}
if(row['A'] && row['A']=="别墅"){
def inputAare = row['F'].toString().toDouble()
def minArea = inputAare + 50
def maxArea = inputAare + 100
if(r.area<minArea && r.area>maxArea){
continue;
}
}
lastRets << r
if(lastRets.size()>=3){
break;
}
}
if(lastRets){
for(def lr : lastRets){
def data = []
data << row['A']
data << row['B']
data << row['C']
data << row['D']
data << row['E']
data << row['F']
data << row['G']
if(dist){
data << dist.address
}else{
data << ""
}
if(dist){
data << dist.name
}else{
data << ""
}
data << ""
data << ""
if(lr.height){
if(lr.height>6){
data << "是"
}else{
data << "否"
}
}else{
data << ""
}
data << lr.floor
data << lr.height
data << lr.fitment
data << ""
data << lr.direction
data << ""
data << lr.build_type
data << lr.area
data << dist.property_life
// def year = dist.create_year
// if(year){
// data << getHouseYear(year.toInteger())//已使用年限
// }else{
// data << ""//已使用年限
// }
data << getHouseYear(dist.create_year , lr.build_date, dist.build_date)
data << getEvalutaionPrice(lr, dist)//评估价(单价)
data << lr.avg_price
data << getDealPrice(dist)//成交单价
data << row['M']
datas << data
}
}
}
}
})
ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据")
}
def getDealPrice(dist){
def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100)
def prices = []
if(json && json.code==200){
def result = json.result
if(result){
for(def r : result){
prices << r.avgPrice
}
}
}
def dealAvgPrice = null
if(prices){
dealAvgPrice = (prices.sum()/prices.size()).toInteger()
}
return dealAvgPrice;
}
def getHouseYear(create_year , house_build_date, dist_build_date){
def year = null
if(house_build_date){
year = getYear(house_build_date)
}else if(dist_build_date){
year = getYear(dist_build_date)
}else if(create_year){
year = create_year
}
if(year){
year = year.toInteger()
def curYear = new Date().getYear()+1900
def modYear = curYear - year
return modYear.toInteger().toString()
}
return null
}
def getYear(finishBuildingDate){
if(finishBuildingDate){
Pattern pat3 = Pattern.compile("\\d{4}");
def matcher3 = pat3.matcher(finishBuildingDate);
if(matcher3.find()){
return matcher3.group()
}
}
return null
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
def addCommId(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['H']){
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
if(dist){
row['L'] = dist._id.toString()
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['L']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['M'] = priceObj.avg_price
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
String city = row['H']//RegionData.formatCity(row['B'])
if(city){
def address = row['C']
// def region = row['C']
println "[match] city:"+city+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
if(jsonObject != null && jsonObject.precise!=-1) {
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("name")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("id")
}
}else{
println row['C']
}
}
})
}
def stat(path){
ExcelUtils.modify(path, { row ->
if(row['L'] && !row['H']){
def city = row['L']
def region = row['M']
def dists = null
if(city && region){
dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"])
}else{
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"])
}
def distLists = []
def villaLists = []
for(def d : dists){
if(d.estate_type2=="别墅"){
villaLists << d
}else{
distLists << d
}
}
row['J'] = dists.size()
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
row['H'] = haveHouseList
row['I'] = haveVillaList
// row['N'] = distLists.size()
// row['O'] = villaLists.size()
}
})
}
def hailanStat(path){
def total = 0
def distTotal = 0
def villaTotal = 0
def haveHouseDistTotal = 0
def haveHouseVillaTotal = 0
def buildStructureTotal = 0
// ExcelUtils.modify(path, { row ->
// if (row['L'] && !row['K']) {
//"海口","三亚","三沙","儋州",
def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
for(def city : citys) {
def dists = null
// if (city && region) {
// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// } else {
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// }
def distLists = []
def villaLists = []
for (def d : dists) {
if (d.estate_type2 == "别墅") {
villaLists << d
} else {
distLists << d
}
}
total = total + dists.size()
// def structureCount = 0
// for(def d : dists){
// if(d.build_structure){
// structureCount ++
// }
// }
// println structureCount
// row['K'] = structureCount
// buildStructureTotal = buildStructureTotal + structureCount
//
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
haveHouseDistTotal = haveHouseDistTotal + haveHouseList
haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList
distTotal = distTotal + distLists.size()
villaTotal = villaTotal + villaLists.size()
// println c +" "+total+" "+buildStructureTotal
// }
}
// })
println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal
}
def findHaveHouseDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(dist.city, dist.region, "sell", 20210101, null, dist.name, dist.alias, dist.estate_type2)
if(!ret){
ret = findThirdPrice(dist)
}
if(ret && ret.size()>0){
rets.add(dist)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findThirdPrice(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("s_date", new BasicDBObject('$gte', 202201))
def dbcursor = AVERAGE_STATISTICS.find(q)
def prices = []
if(dbcursor){
for(def d : dbcursor){
def set = d.keySet();
Iterator it = set.iterator()
while(it.hasNext()){
def key = it.next()
if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){
def price = d[key]
if(price){
prices << price
}
}
}
}
}
return prices;
}
}
@@ -0,0 +1,328 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.apache.lucene.queryparser.classic.QueryParser
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class MingShenPocStat {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
static main(args) {
def task = new MingShenPocStat()
// task.run()
task.officeRun()
}
def officeRun(){
def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx"
def datas = []
ExcelUtils.modify(path, { d ->
if(d['B']){
def office = CommonDatautils.findDistrictById(d['C'])
if(office){
def alias = []
alias << office.name
if(office.alias){
def nameAlias = office.alias
for(def o : nameAlias){
if(o){
alias << o
}
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
def houses = findOfficeHouseCount(office.city, office.region, aliasStr)
if(houses){
if(houses.size()>=2){
houses = houses[0..1]
}
for(def h : houses){
def obj = findOfficeAvgPrice(office)
def rets = []
rets << d['A']
rets << d['B']
rets << ''
rets << h.region
rets << office.address
rets << h.area
rets << h.floor
rets << h.height
def level = office.bed_kind
rets << level
rets << h.fitment
def date = office.finish_building_date
if(date){
rets << date.substring(0, 4)
}else{
rets << ""
}
rets << ""
rets << ""
def price = ""
if(obj && obj.avg_price){
price = obj.avg_price
rets << obj.avg_price
}else{
rets << ""
}
if(price){
rets << (price*h.area).toInteger()
}
datas << rets
}
}
}else{
def rets = []
rets << d['A']
rets << d['B']
datas << rets
}
// def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"])
}
})
ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本")
}
def run(){
def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx"
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1)
//
// ExcelUtils.modify(path, { row ->
// if(row['C']) {
// def total = 0
// def pre = 0
// for(def d : datas){
// if(row['C']==d['A']){
// total = d['B']
// pre = d['H']
// break
// }
// }
// if(total){
// def c = (total*pre/100).toInteger()
// row['E'] = c
// row['F'] = pre+"%"
// }
// }
// })
def totalcount = findHouseTotal("大连")
def count = findHouseCount("大连", "height")
println DataHandleUtils.getDoubleRound(2, count*100/totalcount)
// ExcelUtils.multiThreadingModify(path, { row ->
// if(row['C']) {
//def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"])
// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"])
// def total = districts.size()
// if(total){
// def house_dist_count = findHouseStat(districts)
// def pre = house_dist_count*100/total
// pre = DataHandleUtils.getDoubleRound(2, pre)
// println total+" "+house_dist_count+" "+pre
// }
// def count = findHouseCount(row['C'])
// row['D'] = count
// println row['C']+" "+count
// row['H'] = findDistrictSellStat(row['C'])
// row['G'] = getNewDist(row['C'])
// def totalcount = findHouseTotal(row['C'])
// def total = row['Z']
// def count = findHouseCount(row['C'], "build_date")
// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total)
// println count +" "+row['Q']
// def obj = getFieldPrecent(row['C'])
// if(obj){
// def detail = obj.district_detail
// row['E'] = detail.alias.toDouble()*100
// row['F'] = detail.address.toDouble()*100
// row['G'] = detail.lng.toDouble()*100
// row['H'] = detail.buildType.toDouble()*100
// row['I'] = detail.outPics.toDouble()*100
// row['J'] = detail.developer.toDouble()*100
// row['Q'] = detail.buildDate.toDouble()*100
// row['R'] = detail.buildDate.toDouble()*100
// }
// }
// });
}
def getFieldPrecent(city){
def q = new BasicDBObject()
q.put("city", city)
def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1))
if(rets){
return rets[0]
}
return null
}
def getNewDist(city){
String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' "
def count = null
ElasticUtils.executeSqlQuery(sql,{ data->
count = data.count
})
return count;
}
def findDistrictSellStat(def city){
def q = new BasicDBObject()
q.put("city", city)
def count = district_sell_stat_coll.count(q)
return count
}
def findHouseStat(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(rMap.city, rMap.region, "sell", 20210101,rMap.name, rMap.alias)
if(ret && ret.size()>0){
rets.add(rMap)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findHouseTotal(city){
def count = 0
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouseCount(city, field){
def count = 0
def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null"
ElasticUtils.executeSqlQuery(sql, {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouses(def city, def region, def cat, def startDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
def findOfficeAvgPrice(office){
def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
if(rets){
def obj = rets[0]
return obj
}
return null
}
def findOfficeHouseCount(city, region, names){
def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
return rets
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -0,0 +1,459 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class MingShenPocTest {
def total = 5000
static main(args) {
def task = new MingShenPocTest()
task.run()
}
def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分
// ner(path)//城市,区域带市带区
// match(path)
// addPrice(path)
// addLatLng(path)
// addAroundPrice(path)
// addAuthor(path)
addHouse(path)
}
def structureV2(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A']){
def jsonObj = FangjiaClientUtils.addressStructure(null, row['A'])
if(jsonObj && jsonObj.result){
if(jsonObj.result.province){
row['B'] = jsonObj.result.province
}
if(jsonObj.result.city){
def cityStr = null
if(jsonObj.result.city=="市辖区"){
cityStr = jsonObj.result.province
}else{
cityStr = jsonObj.result.city
}
row['C'] = RegionData.formatCity(cityStr)
}
if(jsonObj.result.district){
row['D'] = jsonObj.result.district
}
}
}
})
}
def ner(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['A'] && !d['B']) {
def json = FangjiaClientUtils.addressStand(null, d['A'])
if(json && json.entities){
def entities = json.entities
for(def m : entities){
if(m.type=="prov"){
d['B'] = m.word
}
if(m.type=="city"){
def cityStr = m.word
if(cityStr){
d['C'] = cityStr//RegionData.formatCity(cityStr)
}
}
if(m.type=="dist") {
if(d['C']){
d['D'] = m.word //RegionData.formatRegion(d['C'], m.word)
}
}
}
}
}
});
}
//可以根据测试数据添加 推荐楼盘名
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C'] && !row['E']){
def city = RegionData.formatCity(row['C'])
def region = null
if(city){
region = RegionData.formatRegion(city, row['D'])
}
def address = row['A']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property"
if(jsonObject != null && jsonObject.precise!=-1) {
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("propertyType")
row['I'] = jsonObject.get("id")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['I']
if(commid && !row['J']){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def house = [:]
house.area = 100
def price = getEvalutaionPrice(house, dist)
if(price){
row['J'] = price
}
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// row['J'] = priceObj.avg_price
// }
}
}
})
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def addLatLng(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) {
def json = BaiduMapUtils.searchByAddress(d['A'])
if(json && json.location){
d['K'] = json.location.lng
d['L'] = json.location.lat
}
}
})
}
def addAroundPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['C'] && d['K']){
def city = RegionData.formatCity(d['C'])
def dist = new BasicDBObject()
dist.city = city
dist.lng = d['K'].toString().toDouble()*10000000000L
dist.lat = d['L'].toString().toDouble()*10000000000L
dist.estateType = "普通住宅"//d['H']
if(dist.lat) {
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
if (list) {
list = sortByDistance(dist, list);
if (list && list.size() > 10) {
list = list[0..9]
}
if (list) {
def prices = []
for (def s : list) {
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if (priceObj) {
prices << priceObj.avg_price
}
}
if (prices) {
def arround_price = (int) (prices.sum() / prices.size())
d['J'] = arround_price
}
}
}
}
}
});
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addAuthor(def path){
def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['A']){
if(index < avg){
row['M'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addHouse(path){
def datas = []
ExcelUtils.multiThreadingModify(path, { row ->
def commId= row['E']
if(commId){
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
if(rets){
def data = []
for(def r : rets){
data << r.city
data << r.region
data << r.district_name
data << r.floor
data << r.height
data << r.fitment
data << r.direction
data << r.build_type
data << r.area
data << r.avg_price
datas << data
}
}
}
}
});
ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据")
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.chongqing
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.io.FileUtils
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
/**
* 3DES加密工具类
* @author qzz
*/
public class ThreeDESUtils {
/**
* 加解密统一编码方式
*/
private final static String ENCODING = "utf-8";
/**
* 加解密方式
*/
private final static String ALGORITHM = "DESede";
/**
*加密模式及填充方式
*/
private final static String PATTERN = "DESede/ECB/pkcs5padding";
/**
* 3DES加密
*
* @param plainText 普通文本
* @param sK 秘钥(24位密码)
* @return
* @throws Exception
*/
public static String encode(String plainText,String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
// 执行加密操作
byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING));
return Base64.getEncoder().encodeToString(encryptData);
}
/**
* 3DES解密
*
* @param encryptText 加密文本
* @return
* @throws Exception
*/
public static String decode(String encryptText, String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.DECRYPT_MODE, secretKey);
// 正式执行解密操作
byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText));
return new String(decryptData, ENCODING);
}
public static void main(String[] args) throws Exception {
def rets = []
def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt")
def index = 0
for(def d : datas){
index++
def row = []
def str = decode(d,"111036369260679051122113")
println str
def id = str.substring(str.length()-4)
row << id
row << str.substring(0, str.indexOf(id))
rets << row
}
ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)")
// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row ->
// //加密
// String str = row['A']
// //解密
// row['B'] = (decode(str,"111036369260679051122113"));
// });
}
public static def readTxt(def file){
def lines = FileUtils.readLines(new File(file))
def ret = [];
for(line in lines) {
ret << line;
}
return ret;
}
}
@@ -755,7 +755,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","şţĹĎźŃÔˇ")
// q.put("name","Íò´ï»ª¸®Ò»ÆÚ")
DBObject keys = new BasicDBObject();
if(fields){
@@ -903,6 +903,36 @@ class CommonDatautils {
}
return null;
}
public static def findSeaweedById(String id, def filelds){
if(!id){
return null;
}
try{
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
DBObject keys = new BasicDBObject();
filelds.each{
keys.put(it, 1);
}
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
}catch(Exception e){
}
return null;
}
public static def findDistrictBySid(String sid){
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -18,7 +18,9 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
//小区月度均价表(带涨跌幅控制)
@BatchOwner(Watcher.GaoJuan)
class DistrictSellStatBatch extends BaseBatch {
@@ -58,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -77,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch {
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
print city +" "+districts.size()
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
@@ -111,8 +111,8 @@ class FzDataResultStat {
def regions = ["뮬짜","꾑�","憩쉭","쐈갛","쯩棺","낀있"];
def datas1 = []
def datas2 = []
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
def row_num = 0
for(def row : rows){
if(row_num>0){
@@ -420,8 +420,8 @@ class HouseExportBatch extends BaseBatch{
row << seawater.name
row << seawater.url
// row << seawater.p_date
row << DateUtils.getRandomDate("2022-06-01", "2022-06-30")
row << seawater.p_date
// row << DateUtils.getRandomDate("2022-06-01", "2022-06-30")
rets << row
}
}
@@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
def rets = []
for(def d : datas){
def arrs = d.split("@")
rets << ["commId":arrs[0], "avg_price":arrs[4]]
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" 不存在"
}
}
//List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
//println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
//def ret = []
@@ -48,29 +75,29 @@ import org.bson.types.ObjectId
// return location;
//}
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5])
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5])
}
def rows = []
map1.each {k,v->
def row = []
def keys = k.split("##")
def lastPrice = map2.get(k)
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
//def map1 = [:]
//for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
//}
//
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
//def map2 = [:]
//for(def s : list2) {
// def row = s.split("@\\|@")
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
//}
//def rows = []
//map1.each {k,v->
// def row = []
// def keys = k.split("##")
// def lastPrice = map2.get(k)
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
//}
//ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
// if(obj){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -51,6 +51,7 @@ class AddAvgpriceBatch {
for(def disKey : res2){
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date())
}
}
@@ -17,15 +17,15 @@ class CityStatExport {
static main(args) {
def task = new CityStatExport();
//task.run()
task.run1()
task.run()
// task.run1()
}
def run(){
// def citys = CommonDatautils.findAllCity();
def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"]
def citys = ["苏州","无锡","南通","徐州","连云港","宿迁","盐城","镇江","青岛","南京","常州","淮安","泰州","扬州","上海"]
def ret = []
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"]
ret << ["城市","小区总量"]
@@ -45,7 +45,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\福建省小区数据统计.xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\小区数据统计.xlsx",ret,"数据")
}
@@ -0,0 +1,52 @@
package com.fangjia.script.gj
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class ShLongShangHangLocation {
static main(args) {
def task = new ShLongShangHangLocation()
task.run()
}
def run(){
def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"]
def map = [:]
for(def r : rets){
def location = getLocation(r)
if(location){
map.put(r, location)
}
}
def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K']
ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row->
if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){
for(def i=0;i<rets.size();i++){
def s = rets.get(i)
def mapLoc = map.get(s)
def location = getLocation("杨浦区"+row['B'])
if(location && mapLoc){
def distance = GeoUtils.calculateDistance(mapLoc.lng.toDouble(), mapLoc.lat.toDouble(), location.lng.toDouble(), location.lat.toDouble())
def title = titleMap.get(i)
row[title] = distance.toInteger()
}
}
println row['B']
}
},0)
}
def getLocation(def address){
def json = GaoDeApiUtils.geo("上海", address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
return null
}
}
File diff suppressed because one or more lines are too long
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-10-01"
static def statDate = "2023-01-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run();
// task.run1()
// task.updateField()
task.fenQiRun()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
def fenQiRun(){
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"])
def distMap = findFiqiDatas(dists)
println c +" "+dists.size() +" "+ distMap.size()
addFieldValue(distMap);
}
}
def addFieldValue(distMap){
def index = 0
distMap.each {k,v->
if(v.master_district_name!="master"){
def masterKey = v.city +"##"+v.region+"##"+v.master_district_name;
// def masterObj = distMap.get(masterKey)
def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district");
if(masterObj){
def udata = new BasicDBObject()
if(!v.virescence_percent && masterObj.virescence_percent){
udata.put("virescence_percent", masterObj.virescence_percent)
}
if(!v.build_area && masterObj.build_area){
udata.put("build_area", masterObj.build_area)
}
if(!v.all_area && masterObj.all_area){
udata.put("all_area", masterObj.all_area)
}
if(!v.cubage_percent && masterObj.cubage_percent){
udata.put("cubage_percent", masterObj.cubage_percent)
}
if(udata){
index ++
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(v._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
println index;
}
def findFiqiDatas(dists){
def distMap = [:]
for(def d : dists){
def master_district_name = d.master_district_name
if(master_district_name && master_district_name!="null"){
distMap.put(d.city+"##"+d.region+"##"+d.name, d);
}
}
return distMap;
}
def updateField(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['Q'] && !row['Q'].contains("接口")){
def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"])
if(dist && !dist.virescence_percent){
def val = row['K']
if(!val){
val = row['J']
}
if(!val){
val = row['M']
}
if(!val){
val = row['O']
}
if(!val){
val = row['G']
}
if(!val){
val = row['E']
}
if(!val){
val = row['F']
}
if(!val){
val = row['H']
}
if(!val){
val = row['I']
}
if(!val){
val = row['L']
}
if(!val){
val = row['N']
}
if(!val){
val = row['O']
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("virescence_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
row['U'] = 1
}
}
}
})
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
if(row['A'] && !row['Q']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
def flag = "名称匹配"
def dist = matchDist(city, region, name)
if(!row['D'] && name.contains("(")){
name = name.replace("(${region})","").replace("(","").replace(")","")
flag = "去括号匹配"
dist = matchDist(city, region, name)
}
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(rets && rets.size()==1){
dist = rets[0]
}
}
//
// if(!dist){
@@ -61,23 +182,21 @@ class AnjukeDistrictMatch {
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
if(dist){
row['Q'] = flag
row['R'] = dist._id.toString()
row['S'] = dist.region
row['T'] = dist.name
}else{
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = "匹配接口"
row['R'] = jsonObject.get("id")
row['S'] = jsonObject.get("district")
row['T'] = jsonObject.get("name")
}
}
}
})
}
@@ -73,7 +73,10 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
@@ -90,10 +93,15 @@ class FangjiaClientUtils {
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
def index = 0
if(houses) {
for (def d : houses) {
def request = d.request
if(request && !d.domain.contains("k8s2.fangjia.com")){
index ++
println "[${index}] "+d.domain +" "+ request
}
}
}
println set
}catch(Exception e){
e.printStackTrace()
}
return total
}
@@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, region)
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
@@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "±´¿ÇÕÒ·¿"
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(detail.total_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -398,13 +401,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("¸£ÖÝ")){
if(e.text().contains("ÖØÇì")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return list
@@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
@@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome {
}
detail.source = "Á´¼ÒÔÚÏß"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(detail.total_price){
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}
}catch(Exception ex){
@@ -470,11 +474,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["¸£ÖÝ"]
def citys= ["ÖØÇì"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
}
}
@@ -0,0 +1,174 @@
package com.fangjia.script.gj.test
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class DongGuangApiTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new DongGuangApiTest()
task.run()
}
def run(){
// def rets = findEsData();
def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv")
println "ES 加载完成"
def datas = evaluation(rets)
println "重估完成"
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true);
println "txt 生成"
exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas)
println "csv 生成"
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("#",-1)
if(row && row.size()>=2){
def s = row[1]
JSONObject result = JSONObject.fromObject(s);
rows << result
}
}
return rows;
}
def evaluation(rets){
def datas = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def dist = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100)
if(ret && ret.avgPrice){
def row = []
row << dist.city
row << dist.district
row << dist.name
row << dist.size
row << ret.city
row << ret.district
row << ret.name
row << ret.avgPrice
datas << row
}else{
def row = []
row << dist.city
row << dist.district
row << dist.name
row << dist.size
datas << row
}
println totalCount.get() +"/"+rets.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return datas;
}
def findEsData(){
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime())
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007713'))
.must(QueryBuilders.termQuery("request_uri", '/property/evaluate'))
.must(date_range_cdate)
def rets = new CopyOnWriteArrayList()
// def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
// if(datas){
// for (int k=0;k<datas.size();k++) {
// def map = datas[k];
// def paramMap = map.parameter_map
// rets << paramMap
// }
// }
ElasticUtils.executeQuery("192.168.0.206", "open_uri_log", queryBuilder, { Map map ->
def paramMap = map.parameter_map
rets << paramMap
if(rets.size()%10000==0){
println "."+ new Date()
}
})
return rets;
}
def exportCsv(path, list){
File file = new File(path);
if (file.exists()){
file.delete();
}
file.append("用户输入城市,")
file.append("用户输入城市区域,")
file.append("用户输入城市地址,")
file.append("用户输入城市面积,")
file.append(System.getProperty("line.separator"));
list.each{
def d = it;
file.append(d[0] + ",","gbk")
file.append(d[1] + ",","gbk")
file.append(d[2] + ",","gbk")
file.append(d[3] + ",","gbk")
if(d[4]){
file.append(d[4]+",","gbk")
}else{
file.append(",")
}
if(d[5]){
file.append(d[5] +",","gbk")
}else{
file.append(",")
}
if(d[6]){
file.append(d[6] +",","gbk")
}else{
file.append(",")
}
if(d[7]){
file.append(d[7]+",","gbk")
}else{
file.append(",")
}
file.append(System.getProperty("line.separator"));
}
}
}
@@ -3,12 +3,15 @@ package com.fangjia.script.gj.test
import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d ->
//})
ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d ->
def info = d['A']
if(info && !d['E']){
def arrs = info.split("!~")
if(arrs && arrs[2] && arrs[3]){
// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]);
// if(rMap) {
// d['E'] = rMap.name
// d['F'] = rMap.id
// }
def address = null
if(d['G']){
address = d['G']
}else{
address = arrs[4]
}
def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
d['H'] = jsonObject.get("city")
d['I'] = jsonObject.get("district")
d['J'] = jsonObject.get("name")
d['K'] = jsonObject.get("id")
}
// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4])
// if(jsonObj && jsonObj.result) {
// def street = null
// def street_number = null
// if (jsonObj.result.street) {
// d['E'] = jsonObj.result.street
// }
// if (jsonObj.result.streetNumber) {
// d['F'] = jsonObj.result.streetNumber
// }
// d['G'] = jsonObj.result.name
// }
// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3])
// if(rets && rets.size()==1){
// def obj = rets[0]
// d['B'] = obj.province
// d['C'] = obj.fjw_city
// d['D'] = obj.fjw_district
// }
}
}
});
//
//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){
// def q = new BasicDBObject()
// q.put("city_code", cityCode.toString().toInteger())
// q.put("district_code", regionCode.toString().toInteger())
// def dbcursor = FANGJIA_AREA_CODE.find(q)
// def rets = []
// for(def d : dbcursor){
// rets << d
// }
// return rets;
//}
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){
@@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi
// }
//})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){
def info = infos[0]
d['D'] = info.province_code
d['E'] = info.area_code
d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
// if(!d['D']){
// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
// if(infos && infos.size()==1){
// def info = infos[0]
// d['D'] = info.province_code
// d['E'] = info.area_code
// d['F'] = info.street_code
// }
}
return rows
}
// }
//})
//
//
//def findTownshipInfo(city, region, coll){
// def q = new BasicDBObject()
// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
//
// def dbcursor = coll.find(q)
// def rows = []
// for(def d : dbcursor){
//// if(d.street_name.contains(region)){
// rows << d
//// }
// }
// return rows
//}
@@ -98,7 +98,7 @@ public class EmailConsumer extends BaseBatch {
}
} else {
println "send error ${errorCount}, sleep ${sleepTime / 1000}s ..."
println "${mail.receiver_email} send error ${errorCount}, sleep ${sleepTime / 1000}s ..."
sleep(sleepTime);
}
}
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");//·¢ËÍ·½
email.setSubject(mail.subject); //±êÌâ
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");
email.setSubject(mail.subject);
@@ -247,6 +247,35 @@ class GaoDeUtils {
return Collections.emptyList()
}
static List<AmapPoi> getAmapById2(id){
def params = [:]
params.id = id
params.output = 'json'
params.key = "66e6771767ab293ace441553c36621c1"
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return Collections.emptyList()
}
/**
* »ñÈ¡ÏêÇé
*/
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "Á´¼Ò"//
def citys = findCrawlCity(source, 202209);
def source = "365ÌÔ·¿"//
def citys = findCrawlCity(source, 202211);
for(def city : citys){
def count1 = findCount(source, city, 202209);
def count2 = findCount(source, city, 202210);
def count1 = findCount(source, city, 202211);
def count2 = findCount(source, city, 202212);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["졍소":"졍소"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
static main(args) {
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202209);
def count2 = findTotalCount(source, 202210);
def count1 = findTotalCount(source, 202212);
def count2 = findTotalCount(source, 202301);
println source +" "+count1 +" "+count2
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.util
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.addrepo.InvokeHttpRequest;
import groovy.json.JsonSlurper
@@ -103,24 +104,20 @@ class BaiduMapUtils {
int recount = 0;
while(!stop && recount<3) {
try{
def ret = ABuYunUtil.doGet(url, null, null, "utf-8");
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj.getInt("status") == 121){
println "key over"
changeAK()
recount++;
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
json = searchByAddress(address);
}else{
def result = jsonObj.result;
if(result){
stop = true
return result;
}else{
stop = true
}
return json.result
}
}catch (Exception e){
e.printStackTrace();
}
return null;
}catch (Exception e){
recount++;
println e.getMessage()
@@ -335,8 +335,15 @@ public class DbUtils{
keys.append("lng2", 1)
keys.append("estate_type2", 1)
DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo->
if(!district._id || dbo._id != district._id){
districts << dbo
if(district.estateType){
if(dbo.estate_type2 == district.estateType){
districts << dbo
}
}else{
districts << dbo
}
}
}
}
@@ -221,6 +221,9 @@ class ElasticUtils {
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
@@ -65,6 +65,8 @@ class MongoUtils {
if (domainUrl == MongoDB_Domain_mongo) {
port == 28017;
}
if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}
@@ -1105,6 +1105,30 @@ public class OpenApiClient {
return null
}
public static JSONObject addressEvaluation(city, region, address, size){
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
params.put("size", String.valueOf(size));
params.put("token", TOKEN);
String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8");
JSONObject result = JSONObject.fromObject(response);
if (result.getInt("code") == 200) {
if (result.containsKey("result")) {
return result.getJSONObject("result")
}
} else if (result.getInt("code") == 500) {
println("µØÖ·¹À¼Ûʧ°Ü:${city} ${address} ${size} ${result}")
}
} catch (Exception e) {
e.printStackTrace();
}
return null
}
public static JSONObject evaluateDistrictPriceByAddress(String city, String address, double size) {
try {
HttpRequest request = new HttpRequest();
@@ -1 +1,18 @@
package com.fangjia.script.wenyan
import com.fangjia.core.ProvinceData
import com.fangjia.script.util.Jms
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.client.MongoCollection
import org.bson.Document
def jms = Jms.create("tcp://jms.db.fangjia.com:61616")
jms.sendMail("1142375881@qq.com", "²âÊÔ", "²âÊÔÄÚÈÝ")
println('1111')
@@ -0,0 +1,43 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007713"), eq("request_uri", "/property/evaluate"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "东莞银行总行", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
// hashSet << result
// } else {
// println("已存在")
// }
}
if (count % 7 == 0) {
hashSet.clear()
}
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) {
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
} else {
@@ -38,4 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
/**
* 冪帠僅蛌華硊<br>
@@ -17,4 +17,6 @@ public interface AverageStatisticsService {
AverageStatistics findSecendAverageStatictics(String city,String region,String name,String category);
Map<String, Object> findFirstAverageStatictics(String city,String region,String name,String category, Date sDate);
AverageStatistics save(AverageStatistics averageStatistics);
}
@@ -198,8 +198,14 @@ public class AverageStatisticsServiceImpl implements AverageStatisticsService {
}
return dataMap;
}
@Override
public AverageStatistics save(AverageStatistics averageStatistics) {
averageStatisticsDao.save(averageStatistics);
return averageStatistics;
}
private boolean isValid(Integer value) {
if (value != null && value > 1000) {
return true;
@@ -7,12 +7,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fangjia.core.util.CacheUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.sql.dao.UserDao;
import com.fangjia.dal.sql.model.User;
import org.sothis.core.util.Closure;
public class UserServiceImpl implements UserService {
private UserDao userDao;
@@ -28,10 +30,15 @@ public class UserServiceImpl implements UserService {
return Collections.emptyList();
}
public User findByUserName(String userName) {
List<QueryParam> params = new ArrayList<QueryParam>(1);
params.add(new QueryParam("userName", userName));
return userDao.findFirstByParams(params);
public User findByUserName(final String userName) {
return CacheUtils.getValue(UserServiceImpl.class, "findByUserName", userName, CacheUtils.CachePolicyEnum.TODAY, new Closure<User, String>(){
@Override
public User execute(String input) {
List<QueryParam> params = new ArrayList<QueryParam>(1);
params.add(new QueryParam("userName", userName));
return userDao.findFirstByParams(params);
}
});
}
public void updateUser(User user) {