diff --git a/Jenkinsfile b/Jenkinsfile index 1b1725b85..37a7f1e92 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,11 +16,15 @@ pipeline { } stage('Maven Build') { steps { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - for (def project : projects) { - dir('deliverables/${project}') { - withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn clean package -Dmaven.test.skip=true' + script { + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + for (def project : projects) { + if (params.PROJECT_NAME == project) { + dir("deliverables/${project}") { + withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { + sh 'mvn clean package -Dmaven.test.skip=true' + } + } } } } @@ -33,7 +37,7 @@ pipeline { def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] for (def project : projects) { if (params.PROJECT_NAME == project) { - dir('projects/${imagesName["project"]}') { + dir("projects/${imagesName[project]}") { withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9012') { sh "docker build -t 192.168.0.19:9011/${imagesName[project]}:${env.BUILD_TAG.toLowerCase()} ." withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9011') { diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java index 3b91e6583..71ff8fb02 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/AverageStatisticsController.java @@ -968,9 +968,6 @@ public class AverageStatisticsController { */ public Object baseAction(AverageStatisticsModel model){ Pager pager = model.getPager(25); - if (StringUtils.isBlank(model.getCityDisplay())) { - return model; - } pager.setTotalRows(avgBaseDao.count(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate())); model.setBaseList(avgBaseDao.findAll(model.getCityDisplay(), model.getRegion(), model.getName(),null, model.getStartDate(), model.getEndDate(), pager)); return model; @@ -987,6 +984,8 @@ public class AverageStatisticsController { list.add("是否加入指导价"); list.add("调查时间"); list.add("调查人"); + list.add("来源"); + list.add("来源小区名"); data.add(list); HSSFWorkbook workbook = new HSSFWorkbook(); HSSFSheet sheet = workbook.createSheet("导入模板"); @@ -994,10 +993,12 @@ public class AverageStatisticsController { HSSFCell cell0 = row.createCell(0); HSSFCell cell1 = row.createCell(1); HSSFCell cell2 = row.createCell(2); - HSSFCell cell3= row.createCell(3); - HSSFCell cell4= row.createCell(4); - HSSFCell cell5= row.createCell(5); - HSSFCell cell6= row.createCell(6); + HSSFCell cell3 = row.createCell(3); + HSSFCell cell4 = row.createCell(4); + HSSFCell cell5 = row.createCell(5); + HSSFCell cell6 = row.createCell(6); + HSSFCell cell7 = row.createCell(7); + HSSFCell cell8 = row.createCell(8); cell0.setCellValue("城市"); cell1.setCellValue("区域"); cell2.setCellValue("小区名"); @@ -1005,6 +1006,8 @@ public class AverageStatisticsController { cell4.setCellValue("是否加入指导价"); cell5.setCellValue("调查时间"); cell6.setCellValue("调查人"); + cell7.setCellValue("来源"); + cell8.setCellValue("来源小区名"); HttpServletResponse response = ActionContext.getContext().getResponse(); response.setCharacterEncoding("UTF-8"); response.setContentType("application/x-download;charset=UTF-8"); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java index 52b1ecb2d..a101d08b2 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java @@ -305,11 +305,11 @@ public class EvaluationAuditController { if(trades!=null && model.getAuditPrice()!=null){ int charge = 0; if(model.getAuditPrice() > 0 && model.getAuditPrice() <= 100){ - charge = 40; + charge = 20; }else if(model.getAuditPrice() > 100 && model.getAuditPrice() <= 300){ - charge = 160; + charge = 70; }else if(model.getAuditPrice() > 300){ - charge = 360; + charge = 80; } trades.setCharge((int) charge); userTradesService.update(trades); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java index 3a51dcc61..5d15317e7 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java @@ -401,11 +401,11 @@ public class YugudanController { double price = model.getPeopleEditPrice(); double charge = 0; if(price > 0 && price <= 100){ - charge = 40; + charge = 20; }else if(price > 100 && price <= 300){ - charge = 160; + charge = 70; }else if(price > 300){ - charge = 360; + charge = 80; } EvaluationAudit audit = evaluationAuditService.findByEvalutionId(String.valueOf(model.getId())); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java index f4594ad70..f782a46f9 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/util/BaseAvgImportServlet.java @@ -1,6 +1,7 @@ package com.fangjia.bkoff.util; import com.fangjia.core.Constants; +import com.fangjia.core.RegionData; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.DateUtil; import com.fangjia.core.util.HttpRequest; @@ -8,11 +9,13 @@ import com.fangjia.core.util.HttpRequestException; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.dao.AvgBaseDao; +import com.fangjia.dal.mongo.model.AverageStatistics; import com.fangjia.dal.mongo.model.AvgBase; import com.fangjia.dal.mongo.model.bkoff.GuidingPrice; import com.fangjia.dal.seaweed.model.District; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.bkoff.GuidingPriceService; +import com.fangjia.service.mongo.fangjia.AverageStatisticsService; import net.sf.json.JSONObject; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; @@ -47,12 +50,12 @@ import java.util.*; public class BaseAvgImportServlet extends HttpServlet { - private static final String[] columns = { "城市", "区域", "小区名", "价格", "是否加入指导价", "调查时间", "调查人" }; + private static final String[] columns = { "城市", "区域", "小区名", "价格", "是否加入指导价", "调查时间", "调查人" , "来源", "来源小区名"}; private AvgBaseDao avgBaseDao; private GuidingPriceService guidingPriceService; private DistrictService districtService; - + private AverageStatisticsService averageStatisticsService; @Override public void init(ServletConfig config) throws ServletException { @@ -62,6 +65,7 @@ public class BaseAvgImportServlet extends HttpServlet { avgBaseDao = (AvgBaseDao) context.getBean("avgBaseDao"); guidingPriceService = (GuidingPriceService) context.getBean("guidingPriceService"); districtService = (DistrictService) context.getBean("districtService"); + averageStatisticsService = (AverageStatisticsService) context.getBean("averageStatisticsService"); } catch (Exception e) { e.printStackTrace(); } @@ -174,7 +178,7 @@ public class BaseAvgImportServlet extends HttpServlet { } private Map checkUploadData(Sheet sheet, String name, long version, Map infoMap, - String realName) { + String realName) { int totalNum = sheet.getLastRowNum(); System.out.println("上传文件共:" + totalNum + "个"); if (totalNum > 5000) { @@ -371,7 +375,8 @@ public class BaseAvgImportServlet extends HttpServlet { } else if (key.equals("价格") && map.get(key) != null) { String val = formatNumer(map.get(key)); if (StringUtils.isNotBlank(val)) { - int x = Integer.parseInt(val); + double dd = Double.parseDouble(val); + int x = (int) dd; avgBase.setPrice(x); } @@ -385,18 +390,58 @@ public class BaseAvgImportServlet extends HttpServlet { Date applyDate = (DateUtil.parseDate(map.get(key), "yyyy-MM-dd")); avgBase.setCreationDate(applyDate == null ? curDate : applyDate); } else if (key.equals("调查人") && map.get(key) != null) { - avgBase.setUserName(map.get(key)); + } else if (key.equals("来源") && map.get(key) !=null){ + avgBase.setSource(map.get(key).replaceAll("\\s", "")); + } else if (key.equals("来源小区名") && map.get(key) != null){ + avgBase.setSourceCommName(map.get(key).replaceAll("\\s", "")); } + avgBase.setRegion(RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion())); avgBase.setCategory("district"); - } - if (avgBase.getIsGuidancePrice() == 1) { - - DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(), + District dis = null; + if(StringUtils.isNotBlank(avgBase.getSourceCommName()) && !avgBase.getSourceCommName().equals(avgBase.getDistrictName())){ + DistrictKey districtKey = new DistrictKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()), avgBase.getDistrictName()); - District dis = districtService.findOneByKey(districtKey, "district"); - GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), avgBase.getRegion(), + dis = districtService.findOneByKey(districtKey, "district"); + if(dis.getAlias() == null){ + dis.setAlias(Collections.singletonList(avgBase.getSourceCommName())); + }else if(!dis.getAlias().contains(avgBase.getSourceCommName())){ + dis.getAlias().add(avgBase.getSourceCommName()); + } + if(dis.getMappingAlias() == null){ + dis.setMappingAlias(Collections.singletonList(avgBase.getSourceCommName())); + }else if(!dis.getMappingAlias().contains(avgBase.getSourceCommName())){ + dis.getMappingAlias().add(avgBase.getSourceCommName()); + } + districtService.save(dis); + } + if(StringUtils.isNotBlank(avgBase.getSource()) && avgBase.getSource().equalsIgnoreCase("GXD")){ + AverageStatistics averageStatistics = averageStatisticsService.findSecendAverageStatictics(avgBase.getCity(), avgBase.getRegion(), avgBase.getDistrictName(), "district"); + if(averageStatistics != null){ + averageStatistics.setGxdEsfPrice(avgBase.getPrice()); + }else{ + averageStatistics = new AverageStatistics(); + averageStatistics.setGxdEsfPrice(avgBase.getPrice()); + averageStatistics.setCategory("district"); + averageStatistics.setCity(avgBase.getCity()); + averageStatistics.setRegion(avgBase.getRegion()); + averageStatistics.setName(avgBase.getDistrictName()); + if(dis != null){ + averageStatistics.setMasterDistrictName(dis.getDistrictName()); + averageStatistics.setModificationDate(new Date()); + } + } + averageStatisticsService.save(averageStatistics); + } + + if (avgBase.getIsGuidancePrice() == 1) { + if(dis == null){ + DistrictKey districtKey = new DistrictKey(avgBase.getCity(), avgBase.getRegion(), + avgBase.getDistrictName()); + dis = districtService.findOneByKey(districtKey, "district"); + } + GuidingPrice price = guidingPriceService.findByBizKey(avgBase.getCity(), RegionData.formatRegion(avgBase.getCity(), avgBase.getRegion()), avgBase.getDistrictName(),avgBase.getCreationDate()); if (price != null) { price.setGuidingPerson(avgBase.getUserName()); @@ -414,7 +459,7 @@ public class BaseAvgImportServlet extends HttpServlet { price.setGuidingPerson(avgBase.getUserName()); price.setGuidingDate(avgBase.getCreationDate()); price.setGuidingValue(avgBase.getPrice()); - price.setEstateType(dis.getEstateType2().indexOf("别墅") > -1 ? "别墅" : "普通住宅"); + price.setEstateType(dis.getEstateType2().contains("别墅") ? "别墅" : "普通住宅"); price.setCat("district"); guidingPriceService.save(price); HttpRequest request = new HttpRequest(); @@ -430,11 +475,10 @@ public class BaseAvgImportServlet extends HttpServlet { } catch (HttpRequestException e) { e.printStackTrace(); } + }else{ + System.out.println("小区不存在"+ avgBase.getDistrictName()); } } - - if(dis !=null) { - } } avgBases.add(avgBase); diff --git a/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl b/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl index 417d20240..dd6935446 100644 --- a/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/averageStatistics/base.ftl @@ -66,6 +66,8 @@ 鏄惁鍔犲叆鎸囧浠� 璋冩煡鏃堕棿 璋冩煡浜� + 鏉ユ簮 + 鏉ユ簮灏忓尯鍚� @@ -94,7 +96,17 @@ <#if it.userName??> ${it.userName} - + + + <#if it.source??> + ${it.source} + + + + <#if it.sourceCommName??> + ${it.sourceCommName} + + diff --git a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl index a3cb6949e..b173ee86a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl @@ -254,6 +254,9 @@ <#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??> 365鎴夸骇 + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + GXD + @@ -345,7 +348,10 @@ 鏂版埧 浜屾墜鎴� - + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + 鏂版埧 + 浜屾墜鎴� + @@ -430,7 +436,7 @@ <#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??> ${it.ganjiNewPrice!} ${it.ganjiEsfPrice!} - + <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??> ${it.yungujiaNewPrice!} ${it.yungujiaEsfPrice!} @@ -443,6 +449,10 @@ ${it.tsfhouseNewPrice!} ${it.tsfhouseEsfPrice!} + <#if (it.gxdNewPrice)?? || (it.gxdEsfPrice)??> + ${it.gxdNewPrice!} + ${it.gxdEsfPrice!} + diff --git a/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl b/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl index efec6b2ea..826290bd0 100644 --- a/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/evaluationAudit/list.ftl @@ -316,11 +316,11 @@ var price = new Number(val)*10000 var charge = 0; if(price > 0 && price <= 1000000){ - charge = 40; + charge = 20; }else if(price > 1000000 && price <= 3000000){ - charge = 160; + charge = 70; }else if(price > 3000000){ - charge = 360 + charge = 80; } if(price != 0){ $(id).html(charge+'鍏�'); diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java index 5b64d35c0..7a9f45104 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java @@ -8,6 +8,6 @@ public interface Constants { String LOG_MONGO_URI = "mongodb://192.168.0.42:52002"; String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007"; String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020"; - String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.19:38080/FangJiaDataService"; + String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService"; } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java index 7b2405df1..da498918c 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java @@ -166,6 +166,8 @@ public class AverageStatistics implements MongoEntity { private Double tsfhouseEsfDiff; private Integer tsfhouseNewPrice; private Double tsfhouseNewDiff; + /**国信达*/ + private Integer gxdEsfPrice; private String level;//价格评级 private Integer lastAvgPrice;//最终价 @@ -1155,4 +1157,13 @@ public class AverageStatistics implements MongoEntity { public void setTsfhouseNewDiff(Double tsfhouseNewDiff) { this.tsfhouseNewDiff = tsfhouseNewDiff; } + + @Column(name = "gxd_esf_price") + public Integer getGxdEsfPrice() { + return gxdEsfPrice; + } + + public void setGxdEsfPrice(Integer gxdEsfPrice) { + this.gxdEsfPrice = gxdEsfPrice; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java index 394785d9d..4d1c1dfbf 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AvgBase.java @@ -30,9 +30,9 @@ public class AvgBase implements MongoEntity{ private String category; private Integer status; private String dateStr; - - private Integer isGuidancePrice; + private String source; + private String sourceCommName; @Id @GeneratedValue @@ -135,4 +135,22 @@ public class AvgBase implements MongoEntity{ public void setDateStr(String dateStr) { this.dateStr = dateStr; } + + @Column(name = "source") + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + @Column(name = "source_comm_name") + public String getSourceCommName() { + return sourceCommName; + } + + public void setSourceCommName(String sourceCommName) { + this.sourceCommName = sourceCommName; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 2ae672241..f998411a0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -24,6 +24,7 @@ import com.fangjia.open.model.*; import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; @@ -85,7 +86,6 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -import org.sothis.web.mvc.annotation.Param; @Authority(role = Role.USER) public class AddressController { @@ -116,6 +116,8 @@ public class AddressController { private static final String GRADE_CONSUME = "消费评分"; public final static OpenConfig CONFIG = OpenConfig.getConfig(); + private static final String ES_USER = "developer"; + private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3"; /** * @deprecated see parse2Action @@ -521,8 +523,65 @@ public class AddressController { } MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param); jsonResult.put("matchResult", matchResult); + if (jsonResult.containsKey("category")) { + if ("office".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "写字楼"); + } else if ("shop".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "商铺"); + } + } } else if (jsonResult != null) { - jsonResult.put("matchResult", new MatchDistrictResult()); + // 判断是否为自然村 + boolean matchedVillage = false; + try { + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null; + String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null; + String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null; + String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null; + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + break; + } + } + } + } + } + } + } catch(Exception ex) { + ex.printStackTrace(); + } + if (matchedVillage == false) { + jsonResult.put("matchResult", new MatchDistrictResult()); + } } resultModel.setResult(json.get("result")); mav.setModel(resultModel); @@ -918,6 +977,60 @@ public class AddressController { return ret; } + private JSONObject addTerm(String name, String value){ + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put(name, value); + term.put("term", match); + return term; + } + + /*** + * 获取自然村 + * @param city + * @param region + * @param name + * @return + */ + private List findVillage(String city, String region, String name) { + JSONObject reqBody = new JSONObject(); + reqBody.put("from", 0); + reqBody.put("size", 5); + JSONObject query = new JSONObject(); + JSONObject bool = new JSONObject(); + JSONArray must = new JSONArray(); + must.add(addTerm("city", city)); + must.add(addTerm("region", region)); + must.add(addTerm("name", name)); + bool.put("must", must); + query.put("bool", bool); + reqBody.put("query", query); + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic "+authenticationEncoding+""); + try { + String response = request.postWithBody("http://192.168.0.45:9200/village/_search", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ + return Collections.emptyList(); + } + JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); + List results = new ArrayList(); + for(int i=0 ;i < hits.size(); i++){ + JSONObject jsonObject = hits.getJSONObject(i); + results.add(jsonObject); + } + return results; + + } catch (HttpRequestException e) { + e.printStackTrace(); + } + return Collections.emptyList(); + } + /** * 加载测试数据 * diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 3a03aa8d1..800a2360d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -454,9 +454,7 @@ public class WeiXinEvaluationController { evaluation.setCity(district.getCity()); evaluation.setRegion(district.getRegion()); evaluation.setDistrictName(district.getName()); - if(StringUtils.isBlank(evaluation.getAddress())){ - evaluation.setAddress(district.getAddress()); - } + evaluation.setAddress(district.getAddress()); evaluation.setUserId(userId); evaluation.setDateCreated(new Date()); evaluation.setLastUpdated(new Date()); @@ -491,8 +489,8 @@ public class WeiXinEvaluationController { if(fjtUser!=null && fjtUser.getRole()==17) { - System.out.println("渤海地址搜索 "+evaluation.getAddress()); - List bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getAddress()); + System.out.println("渤海地址搜索 "+evaluation.getCertificateAddress()); + List bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getCertificateAddress()); JSONObject o = null; if(bhDists.size()==0){ bhDists = BHBankUtils.getDistrictByBhbank(district.getName()); @@ -545,7 +543,7 @@ public class WeiXinEvaluationController { if(StringUtils.isBlank(usersDetail.getMobile())){ usersDetail.setMobile(fjtUser.getEmail()); } - System.out.println("渤海询价搜索 "+evaluation.getAddress()); + System.out.println("渤海询价搜索 "+evaluation.getCertificateAddress()); JSONObject bhApiResult = BHBankUtils.getInquirePrice(model, bhMatchMap, usersDetail, evaluation.getTotalPrice()); if(bhApiResult!=null && (Integer)bhApiResult.get("errCode")==0){ System.out.println("渤海询价搜索结果 "+bhApiResult.toString()); @@ -1411,11 +1409,16 @@ public class WeiXinEvaluationController { } if(uploadfiles.size()>0){ JSONObject jsonObj = BHBankUtils.inquireUploadFiles2(usersDetail.getName(),model.getQueryId(), model.getFileClass(), uploadfiles); - if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) { - mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg"))); + if(jsonObj!=null){ + if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) { + mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg"))); + }else{ + mav.setModel(BasicModel.buildServerErrorModel().setMsg(("文件上传成功!"))); + } }else{ - mav.setModel(BasicModel.buildServerErrorModel().setMsg(("文件上传成功!"))); + mav.setModel(BasicModel.buildServerErrorModel().setMsg("上传失败!")); } + } } catch (Exception e) { e.printStackTrace(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java index 5733be72d..eff33a38f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuthenticationInterceptor.java @@ -213,7 +213,7 @@ public class AuthenticationInterceptor implements Interceptor { if (null == apiUser) { return ret; } - if(apiUser.getDeadTime() != null && new Date().after(apiUser.getDeadTime())){ + if(apiUser.getDeadTime() != null && DateUtil.truncateDay(new Date()).after(apiUser.getDeadTime())){ return new HttpCode(OpenHttpCodeEnum.USER_EXPIRED); } if(!isOldUser(apiUser) && isExpired(token)){ diff --git a/projects/fangjia-open/src/main/webapp/static/js/select.js b/projects/fangjia-open/src/main/webapp/static/js/select.js index b7c01d2f6..1202c4ad6 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/select.js +++ b/projects/fangjia-open/src/main/webapp/static/js/select.js @@ -1,9 +1,7 @@ // JavaScript Document - /* 閫夋嫨鎿嶄綔 */ - // 閫夋嫨鎿嶄綔-鍦板潃 -$(document).on('click', '.create-actions00', function () { +/*$(document).on('click', '.create-actions00', function () { var buttons1 = [ { text: '璇烽�夋嫨鍦板潃', @@ -31,6 +29,7 @@ $(document).on('click', '.create-actions00', function () { var groups = [buttons1, buttons2]; $.actions(groups); }); + */ // 閫夋嫨鎿嶄綔-鎴垮瀷-瀹� $(document).on('click', '.create-actions01', function () { diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 68519a79b..5160cbd1a 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -47,13 +47,17 @@
  • -
    -
    -
    鍦板潃锛�
    - -

    璇烽�夋嫨鍦板潃

    +
    浜ц瘉鍦板潃锛�
    +
    +
    +<#--
    --> +<#--
    鍦板潃锛�
    --> +<#-- --> +<#--

    璇烽�夋嫨鍦板潃

    --> +<#--
    -->
    锛堝繀濉」锛�
    diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index 6c6e97d28..a64bb2f90 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -34,7 +34,7 @@

    ${fjtEvaluation.districtName!}

    -

    ${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("鍙�")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}鍙�${fjtEvaluation.roomNum!}瀹� 閲嶆柊璇勪及 >

    +

    <#if fjtEvaluation.certificateAddress??>${fjtEvaluation.certificateAddress!}<#else>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("鍙�")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}鍙�${fjtEvaluation.roomNum!}瀹� 閲嶆柊璇勪及 >

    *璇风‘璁や笌浜ц瘉鍦板潃涓�鑷�

    <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 9469b6d02..811ec290c 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -171,6 +171,11 @@ ojdbc14 10.2.0.4.0 + + org.postgresql + postgresql + 42.2.1 + com.yahoo.platform.yui yuicompressor diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index 7a641d018..ca14ea514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -17,8 +17,8 @@ class AddThirdPartyPrice { } def run(){ -//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关" - def citys = ["林芝"] +// + def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"] def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 066318e29..90803d4ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import groovy.sql.Sql import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommbuildtypeBatch(), args); } @@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"]) - def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); +// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) println city +" "+rows.size() @@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); - println "txt 生成" + println "txt 生成->导Oracle" + + saveToPgSql(rets) } @@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } return rets; } + + + def saveToPgSql(rows){ + def sqls = [] + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + println "start: "+new Date() + def index =0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index bfce0b81b..0177411a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -28,7 +29,10 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -37,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -71,7 +74,9 @@ class BigdataCommunityBatch extends BaseBatch{ allDatas << headers allDatas.addAll(rows) FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" + println "txt 生成->导Oracle" + + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } @@ -169,7 +174,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date) ret << CcbCodeUtils.parseHouseUse(dist.estate_type2) ret << dist.household_count - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger() ret << dist.status ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") ret << "pa101" @@ -222,4 +227,34 @@ class BigdataCommunityBatch extends BaseBatch{ } return ret; } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 28c543e60..35c9912a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataDealBatch(), args); } @@ -63,7 +66,7 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } -// println "curDate:"+curDate + println "curDate:"+curDate // for(def city : citys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); @@ -86,7 +89,6 @@ class BigdataDealBatch extends BaseBatch{ println city +" "+youdaDatas.size() } - def trades = getCityData(city, statMap, ppeDists) if(trades){ def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") @@ -105,8 +107,10 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -634,7 +638,8 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd") def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) @@ -680,4 +685,35 @@ class BigdataDealBatch extends BaseBatch{ latch.await() println city +" "+trades.size()+" "+new Date().toString() } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index d89e8501e..82baa96ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -57,15 +58,22 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataForsaleBatch(), args); } @Override public void process(Object args) { + //目前查询库中是否已存在,用的是Oracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -74,8 +82,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 @@ -102,6 +110,33 @@ class BigdataForsaleBatch extends BaseBatch{ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate) + println city + "(原)房源总量:"+retMap.size() + + def allDatas = retMap.values() + allDatas = repeatHouseId(allDatas) + println city + "(新)房源总量:"+allDatas.size() + + + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + } + + + // def allDatas = createSellDatas(city, retMap.values()); + + def headers = [] +// if(index == 1){ +// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] +// } + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); + println "txt文件生成->手动导入到Oracle " + + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() @@ -116,7 +151,8 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -166,21 +202,38 @@ class BigdataForsaleBatch extends BaseBatch{ }) } latch.await() + return retMap + } - def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + + def repeatHouseId(allDatas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(allDatas.size()) + def rets = new CopyOnWriteArrayList() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + + def ids = new HashSet() + for(def h : rets1){ + ids << h.houseId + } + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment if(seawater.p_date instanceof String){ seawater.p_date = seawater.p_date.toInteger() @@ -599,10 +886,23 @@ class BigdataForsaleBatch extends BaseBatch{ }else{ rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) } rets << rowMap + +// if(houseIds.contains(seawater.id)){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + +// def count = isExistFromHouse(seawater.id) +// if(count){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } } } @@ -758,12 +1058,6 @@ class BigdataForsaleBatch extends BaseBatch{ } - /* - * 生成一个用不重复的ID - */ - def GenNonDuplicateID(randomLength){ - return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) - } @@ -786,14 +1080,56 @@ class BigdataForsaleBatch extends BaseBatch{ println ids.size() } - def getDistrictIds(def cityid){ - def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'"; + def getAllHouseIds(def commid, def cityCode){ + def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'"; - def distIds = new HashSet() + def houseIds = new HashSet() sqlInstance.eachRow(sql,{it-> - distIds << it.commid + houseIds << it.FORSALEID }); - return distIds; + return houseIds; + } + + + def isExistFromHouse(def houseId){ + def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; + + def count = null + sqlInstance.eachRow(sql,{it-> + count = it.count + }); + return count; + } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 517f8c44c..1d9ae5347 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,8 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] +// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -52,7 +53,7 @@ class CcbCommonUtils { def countStr = args[0] if(countStr=="344"){ citys = CcbCommonUtils.citys_all - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="92"){ citys = CcbCommonUtils.citys_92 @@ -60,11 +61,13 @@ class CcbCommonUtils { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="252"){ citys = CcbCommonUtils.citys_252 - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ - citys = CcbCommonUtils.citys_92 + citys = CcbCommonUtils.citys_all + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } println "运行城市数量:"+citys.size() return citys; @@ -422,7 +425,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","云山尚城") +// q.put("name","鸣翠蓝湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 2a9ad549f..81958589e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,13 +7,13 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86" + static def ftpServer = "192.168.0.86"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" static def ftpPath = "/home/ccb_evaluation/output/" - static def ftpPath_rh = "/home/ccb_rh/output/" + static def ftpPath_rh = "/home/ccb_ronghe/output/" static SFTP sftp = new SFTP(); static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); @@ -70,16 +70,26 @@ class CcbFtpTestUtils { } - static def listFileNames(month){ + static def listFileNames(type, month){ //上传到ftp sftpChannel = reConnect(); - Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + Vector files = null; + if(type && type=="rh"){ + files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel); + }else{ + files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + } return files } - static def getFileLength(String path, String month){ + static def getFileLength(String type, String path, String month){ sftpChannel = reConnect(); - def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + def size = 0; + if(type && type=="rh"){ + size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel) + }else{ + size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + } return size } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index a96478424..3123dc829 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -27,6 +27,7 @@ class DealPriceDataProcess { // task.addLianJiaPrice(); task.deleteThirdPartyData() // task.delDealByPe() +// task.delDealData() } def delDealByPe(){ @@ -370,20 +371,33 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys_92 - def month = 202209 + def citys = CcbCommonUtils.citys_all + def month = 202212 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202210) + def rets = findPeGteThirdParty(city, 202301) def total = 0 println city +" "+rets.size() + if(rets){ + set.add(city) + } for(def m : rets){ def cityId = cityCodeDatas.get(city) def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } } + println set.toString() + } + + def findThirdCount(sqlInstance236, sql){ + def count = 0 + sqlInstance236.eachRow(sql,{r-> + count = r.count() + }); + return count } def findPeGteThirdParty(cityName, yearmonth){ @@ -398,4 +412,25 @@ class DealPriceDataProcess { return rets } + def delDealData(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d -> + if(d['A']){ + def count = delDealDataByid(d['A'],d['K'].toString().toDouble(), d['H'].toString().toDouble().toInteger()) + println d['B']+" "+count + } + }) + } + + def delDealDataByid(commid, area, price){ + def sql = "select count(*) as count from bigdata_deal_gj where commid='${commid}' and buildarea=${area} and DEALUNITPRICE=${price} " + + "and dealdate>=TO_DATE('2022-11-01','yyyy-MM-dd')" +// sqlInstance236.executeUpdate(sql) + def count = 0 + sqlInstance236.eachRow(sql,{r-> + count = r.count + }); + return count + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 71c378532..4df84ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -13,13 +13,15 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//估值 新版 导出所有数据 class ExportOracleAllDataTest { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() def citys = null + def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; static main(args) { def task = new ExportOracleAllDataTest() @@ -41,11 +43,14 @@ class ExportOracleAllDataTest { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] - //不会变 10.25 + +// exportBigdataForsale(new_dists) //2022-07-19 +// +// //不会变 10.25 // exportEvaluationRateData() // exportBigdataPoiData()// // exportBigdataDistrictData()// - +// // exportGuidedPrice()//随人工改变// // exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // @@ -53,29 +58,27 @@ class ExportOracleAllDataTest { // exportBigdataCommunity()// // exportCommunityStaging()//没cityid // exportCommunityBuileType()// - - exportThirdPartyPrice2(new_dists)// -// exportBigdataForsale(new_dists) //2022-07-19 +// +// exportThirdPartyPrice2(new_dists)// // exportBigdataDeal( new_dists) //2022-07-01 -// -// //第三方网站权重表 +//// +//// //第三方网站权重表 // exportWebsiteRate(); -// -// //计算结果相关 +//// +//// //计算结果相关 // exportAvmEvaluationGJNB()// // exportAVMEvaluationItemGJNB()// // // exportCityPrice()// // exportRegionPrice()// -// -// exportCommunityBasePriceGj()// -// exportCommunityBasePrice()// -// + + + exportCommunityBasePrice()// + // exportCommunityPdescribeGj()// // exportCommCaseDetailGj()// - def curDate = curMonth.toString()+"28" - // ok文件 +// // ok文件 exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); @@ -100,7 +103,6 @@ class ExportOracleAllDataTest { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" @@ -131,7 +133,6 @@ class ExportOracleAllDataTest { allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" @@ -166,7 +167,6 @@ class ExportOracleAllDataTest { allRows << data }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" @@ -194,7 +194,6 @@ class ExportOracleAllDataTest { data << r.SOURCECODE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" @@ -228,8 +227,6 @@ class ExportOracleAllDataTest { }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" @@ -267,8 +264,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" @@ -307,8 +302,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -325,7 +318,7 @@ class ExportOracleAllDataTest { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd') sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -344,8 +337,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -390,7 +381,6 @@ class ExportOracleAllDataTest { } } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" @@ -511,7 +501,6 @@ class ExportOracleAllDataTest { data << r.SOURCECODE.replaceAll(~'\\r', '') allRows << data }) - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -528,7 +517,6 @@ class ExportOracleAllDataTest { def exportThirdPartyPrice2(new_dists) { - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -636,7 +624,6 @@ class ExportOracleAllDataTest { }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" @@ -652,7 +639,6 @@ class ExportOracleAllDataTest { def exportBigdataForsale(new_dists){ //["北京","上海"] - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" @@ -669,22 +655,24 @@ class ExportOracleAllDataTest { " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + "where cityId='${cityCode}' and distid not in ${newstr}"; - def rets1 = getHouseDatas(sql1) - allRows.addAll(rets1) + def rets1 = getHouseDatas(sql1, fileName) + if(rets1){ + allRows.addAll(rets1) + } + }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' "; - allRows = getHouseDatas(sql) + allRows = getHouseDatas(sql, fileName) } - if(allRows.size()>0){ - index = index + 1 - if(index == 1){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - }else{ + if(allRows && allRows.size()>0){ +// index = index + 1 +// if(index == 1){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// }else{ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - } - +// } } } @@ -694,7 +682,7 @@ class ExportOracleAllDataTest { " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + "from bigdata_forsale_gj where distid in ${newstr}" - def rets2 = getHouseDatas(sql2) + def rets2 = getHouseDatas(sql2, fileName) if(rets2.size()>0){ FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); } @@ -704,7 +692,7 @@ class ExportOracleAllDataTest { println "BIGDATA_FORSALE_GJ 上传Ftp完成......" } - def getHouseDatas(sql){ + def getHouseDatas(sql, fileName){ def allRows = [] def index = 0 sqlInstance.eachRow(sql,{r-> @@ -764,18 +752,23 @@ class ExportOracleAllDataTest { data << r.SOURCECODE data << r.ATTACH allRows << data -// if(allRows.size()%200000==0){ -// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); -// allRows.clear() -// } + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } }); - return allRows + + if(allRows.size>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } +// return allRows + return null } def exportAvmEvaluationGJNB(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -840,7 +833,6 @@ class ExportOracleAllDataTest { } def exportAVMEvaluationItemGJNB(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -894,6 +886,14 @@ class ExportOracleAllDataTest { def exportCommunityBasePrice(){ def allRows = [] + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; @@ -919,18 +919,19 @@ class ExportOracleAllDataTest { data << r.UPDATETIME data << r.SOURCECODE allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true); + allRows.clear() + } }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", true); } - def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) @@ -939,6 +940,11 @@ class ExportOracleAllDataTest { def exportCommunityBasePriceGj(){ def allRows = [] + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; @@ -964,15 +970,17 @@ class ExportOracleAllDataTest { data << r.UPDATETIME data << r.SOURCECODE allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); } - def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" @@ -998,7 +1006,6 @@ class ExportOracleAllDataTest { allRows << data }); } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1031,9 +1038,6 @@ class ExportOracleAllDataTest { allRows << data }); } - - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -1047,7 +1051,6 @@ class ExportOracleAllDataTest { def exportCommunityPdescribeGj(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" @@ -1102,11 +1105,19 @@ class ExportOracleAllDataTest { data << r.SATUR_RATIO data << r.NEIGHBOR_PRICE allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } }); } println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } println "community_p_describe_gj 数据文件输出完成......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" @@ -1114,18 +1125,16 @@ class ExportOracleAllDataTest { def exportCommCaseDetailGj(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" } def headers = [] - + def allRows = [] def index = 0 for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'"; - def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -1135,15 +1144,16 @@ class ExportOracleAllDataTest { data << r.PRICE_DATE data << r.CREATE_DATE allRows << data - }); - if (allRows && allRows.size() > 0) { - index = index + 1 - if (index == 1) { - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - } else { + if(allRows.size()%200000==0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() } - } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() } println "comm_case_detail_gj 数据文件输出完成......" CcbFtpTestUtils.ftpUpload(null, fileName, curDate) @@ -1151,7 +1161,6 @@ class ExportOracleAllDataTest { } def exportLastFile(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" @@ -1161,7 +1170,6 @@ class ExportOracleAllDataTest { } def exportOkFile(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" @@ -1171,15 +1179,14 @@ class ExportOracleAllDataTest { } def exportInOk(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - Vector files = CcbFtpTestUtils.listFileNames(curDate) + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] while(fileVec.hasNext()){ ChannelSftp.LsEntry file = fileVec.next() if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ - fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) } } def fileName = "/ccb_evaluation/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 22861ba1d..b25c4efbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -43,7 +43,8 @@ class ExportOracleData { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 之前按publish_date导出 + exportBigdataForsale(lastMonthStr +"-18", new_dists) //现在按updatetime导出 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 @@ -659,13 +660,13 @@ class ExportOracleData { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index a368aeb87..2c727edfa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -12,9 +12,10 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//导出估值 增量数据 新版 class ExportOracleDataTest { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() @@ -45,7 +46,7 @@ class ExportOracleDataTest { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataForsale(lastMonthStr +"-18", new_dists) //2022-07-19 按更新时间导出数据 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 @@ -58,7 +59,7 @@ class ExportOracleDataTest { exportCityPrice(curMonth)//无城市区分 exportRegionPrice(curMonth)//无城市区分 - exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 exportCommunityBasePrice(curMonth)//无城市区分 exportCommunityPdescribeGj(curMonth)//无城市区分 @@ -658,13 +659,13 @@ class ExportOracleDataTest { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ @@ -1149,14 +1150,14 @@ class ExportOracleDataTest { def exportInOk(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - Vector files = CcbFtpTestUtils.listFileNames(curDate) + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] while(fileVec.hasNext()){ ChannelSftp.LsEntry file = fileVec.next() if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ - fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) } } def fileName = "/ccb_evaluation/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index b37ee5b94..513e7025c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -13,6 +13,8 @@ class ExportOracleEvaluationData { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + static main(args) { def task = new ExportOracleEvaluationData() task.run(args) @@ -29,7 +31,7 @@ class ExportOracleEvaluationData { exportCityPrice(curMonth) exportRegionPrice(curMonth) - exportCommunityBasePriceGj(curMonth)//已合下面的合并 + // exportCommunityBasePriceGj(curMonth)//已合下面的合并 exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) @@ -45,49 +47,53 @@ class ExportOracleEvaluationData { } def exportAvmEvaluationGJNB(yearMonth){ - def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + - "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + - "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.COMMUNITY_NUM - data << r.HAS_PRICE_COMMUNITY_NUM - data << r.PRICE_COVERAGE - data << r.ARTIFICIAL_CALC_NUM - data << r.ARTIFICIAL_CALC_COVERAGE - data << r.SAMPLE_CALC_NUM - data << r.SAMPLE_CALC_COVERAGE - data << r.ANN_CALC_NUM - data << r.ANN_CALC_COVERAGE - data << r.TEST_NUM - data << r.TEST_COMM_NUM - data << r.TEST_COMM_COVERAGE - data << r.PPE15_COVERAGE - data << r.RATE_MEDIAN - data << r.COEFFICIENT - data << r.DEAL_TEST_CASE_NUM - data << r.DEAL_TEST_COMM_NUM - data << r.DEAL_TEST_COMM_COVERAGE - data << r.DEAL_PPE15_COVERAGE - data << r.DEAL_COEFFICIENT - data << r.DEAL_RATE_MEDIAN - data << r.THIRD_TEST_CASE_NUM - data << r.THIRD_TEST_COMM_NUM - data << r.THIRD_TEST_COMM_COVERAGE - data << r.THIRD_PPE15_COVERAGE - data << r.THIRD_COEFFICIENT - data << r.THIRD_RATE_MEDIAN - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth} and CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" @@ -108,38 +114,43 @@ class ExportOracleEvaluationData { } def exportAVMEvaluationItemGJNB(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + - "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.CONFIDENCE - data << r.TARGET_AVG_PRICE - data << r.DEALDATE - data << r.UNIT_PRICE - data << r.BUILDAREA - data << r.FITMENT - data << r.HOUSEASPECT - data << r.FLOORNUM - data << r.FLOORCNT - data << r.DATA_TYPE - data << r.RATE - data << r.PE - data << r.CREATETIME - allRows << data - }); + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" @@ -158,31 +169,35 @@ class ExportOracleEvaluationData { } def exportCommunityBasePrice(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.FAKE_RATE - data << r.DIFF_RATE - data << r.CONFI_RATE - data << r.CORR_RATE - data << r.CONFIDENCE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" @@ -201,34 +216,39 @@ class ExportOracleEvaluationData { } def exportCommunityBasePriceGj(yearMonth){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.AVG_PRICE - data << r.CALC_TYPE - data << r.FAKE_RATE - data << r.DIFF_RATE - data << r.CONFI_RATE - data << r.CORR_RATE - data << r.CONFIDENCE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -240,21 +260,24 @@ class ExportOracleEvaluationData { } def exportCityPrice(month){ - def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.PRICE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month} AND CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -269,23 +292,27 @@ class ExportOracleEvaluationData { } def exportRegionPrice(month){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.ID - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.YEAR_MONTH - data << r.CALC_AVG_PRICE - data << r.PRICE - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month} AND CITYID='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" @@ -301,68 +328,67 @@ class ExportOracleEvaluationData { def exportCommunityPdescribeGj(month){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } def headers = [] -// -// for(def city : citys) { -// def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; -// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMUNITY_ID - data << r.CITYID - data << r.CITY_NAME - data << r.DISTRICT_CODE - data << r.DISTRICT_NAME - data << r.COMMUNITY_NAME - data << r.PQ_NAME - data << r.BASE_PRICE - data << r.CONFIDENCE_LEVEL - data << r.CNT - data << r.MIN_PRICE - data << r.MEDIANPRICE - data << r.MEANPRICE - data << r.MAX_PRICE - data << r.TIME - data << r.FIRSTDATAT - data << r.THIRDDATA - data << r.RELATIVEERROR - data << r.SKEWNESS - data << r.PQ_PRICE_RANGE - data << r.DISTRICT_PRICE_RANGE - data << r.DESCRIPTION - data << r.PRIV_BASE_PRICE - data << r.IS_ARTTFICIAL - data << r.PRIV_FINAL_PRICE - data << r.CREATE_DATE - data << r.PRICE - data << r.PRICE_SOURCE - data << r.CASE_MD_PRICE - data << r.CASE_AVG_PRICE - data << r.INDEX_LABEL - data << r.MAIN_PROPERTY - data << r.COMMTYPE - data << r.NEW_DEAL_PRICE - data << r.NEW_DEAL_DATE - data << r.PRICE_MOM_RATIO - data << r.MAX_MOM_RATIO - data << r.PRICE_YOY_RATIO - data << r.SATUR_RATIO - data << r.NEIGHBOR_PRICE - allRows << data - }); + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + } println allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); -// } + println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" @@ -370,8 +396,6 @@ class ExportOracleEvaluationData { def exportCommCaseDetailGj(month){ -// def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" @@ -381,8 +405,8 @@ class ExportOracleEvaluationData { def headers = [] for(def city : citys){ - def cityCode = cityCodeDataMap.get(city) - def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month} AND CITYID='${cityCode}'"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index b28516075..4d8b7fa02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -29,7 +29,9 @@ class FjwPricePkCcbPrice { // task.updatePrice() // task.addFjwPrice() // task.addCcbPrice() - task.addMarketPrice() + + task.addCcbPriceFromGuided() +// task.addMarketPrice() } def addMarketPrice(){ @@ -66,6 +68,24 @@ class FjwPricePkCcbPrice { }) } + def addCcbPriceFromGuided(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\人工指导价2.xls",{d-> + if(d['B'] && !d['O']){ + def cityId = d['D'] + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) + if(map){ + d['O'] = map.avg_price + d['P'] = map.calc_type + d['Q'] = map.credit + } + } + } + }) + } + def updatePrice(){ def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index bd64cc3f8..b6b171c81 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -32,6 +33,10 @@ class GuiDingPriceBatch extends BaseBatch{ def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { runBatch(new GuiDingPriceBatch(), args); } @@ -47,7 +52,8 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-23") +// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -76,10 +82,10 @@ class GuiDingPriceBatch extends BaseBatch{ def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); -// def rets = [] -// rets << headers -// rets.addAll(rows) -// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") + saveToPgSql(rows) + + //更新静态数据表 +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } @@ -210,5 +216,43 @@ class GuiDingPriceBatch extends BaseBatch{ return distMap; } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_guided_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(300, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + pgsqlInstance.executeUpdate(sql) + } + + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy new file mode 100644 index 000000000..f2a564cb7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -0,0 +1,20 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +class PgsqlCommUtils { + + public static def database = "warehouse" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index ed95e4cc6..899243d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import groovy.sql.Sql import org.bson.types.ObjectId /** @@ -18,6 +19,9 @@ import org.bson.types.ObjectId @BatchOwner(Watcher.GaoJuan) class StagingDistrictBatch extends BaseBatch{ + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new StagingDistrictBatch(), args); } @@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{ //ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + saveToPgSql(rows) + } def createCityData(def dists, def fenQiMap){ @@ -86,4 +92,34 @@ class StagingDistrictBatch extends BaseBatch{ } return fenQiMap; } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13]) + ps.toString() + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d36f08cc1..f0a1f1448 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,6 +45,139 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) +//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + +//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d-> +// if(d['J']){ +// def cityId = cityCodeDataMap.get(d['H']) +// def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) +// println d['H'] +" "+d['J']+ map.toString() +// if(map){ +// d['Q'] = map.credit +// d['R'] = map.month +// d['U'] = map.price +// } +//// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) +//// if(thirdMap){ +//// d['S'] = thirdMap.price +//// d['T'] = thirdMap.month +//// } +// } +// +//}) + +def findCcbPriceCredit(sqlInstance, cityid, commid){ + def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.year_month + map.credit = r.CONFIDENCE + map.price = r.avg_price + }); + return map +} + +def findThirdPriceBySource(sqlInstance, cityid, commid){ + def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" + + "order by price_date desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.price_date + map.price = r.community_price + }); + return map +} + + + + + +//for(def c : citys){ +// def cityCode = cityCodeDataMap.get(c) +// println c +" "+cityCode +// def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) +//// +// checkNoPriceDist(sqlInstance236, priceDists, cityCode) +//} + +def checkNoPriceDist(sqlInstance236,priceDists, cityid){ + def commids = [] + def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0"; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + + for(def commid : commids){ + if(!priceDists.contains(commid)){ + updateDistStatus(sqlInstance236, commid) + } + } + return commids +} + + +def updateDistStatus(sqlInstance236, commid){ + def sql = "update bigdata_community_gj set status=1 where commid='${commid}'"; + sqlInstance236.executeUpdate(sql) +} +def findAllPriceDist(sqlInstance236, cityid, yearMonth){ + def commids = [] + def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} "; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + return commids +} +//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] +// +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def rets = [] +//for(def c : citys){ +// def citycode = cityCodeDataMap.get(c) +// def datas = findAllData(sqlInstance236, citycode) +// def totalCount = datas.size() +// def landareaCount = 0; +// def buildareaCount = 0 +// def greeningrateCount = 0 +// def plotratioCount = 0 +// for(def m : datas){ +// if(m.landarea){ +// landareaCount ++ +// } +// if(m.buildarea){ +// buildareaCount++ +// } +// if(m.greeningrate){ +// greeningrateCount++ +// } +// if(m.plotratio){ +// plotratioCount++ +// } +// } +// +// def landAreaPre = landareaCount*100/totalCount +// def buildAreaPre = buildareaCount*100/totalCount +// def greeningratePre = greeningrateCount*100/totalCount +// def plotratioPre = plotratioCount*100/totalCount +// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] +//} +// +//ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率") + +def findAllData(sqlInstance236, citycode){ + def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'" + def rets = [] + sqlInstance236.eachRow(sql,{r-> + rets << ["landarea":r.landarea, "buildarea":r.buildarea, "greeningrate":r.greeningrate, "plotratio":r.plotratio] + }); + return rets + +} + + + ////def url = "https://sh.anjuke.com" ////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") ////println s @@ -447,23 +580,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business ////},0) // -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrictById(d['D']) - if(dbo){ - def udata = new BasicDBObject() -// udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println dbo._id.toString() +" 没找到" - } - } -},0) +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> +// if(d['A']){ +// def dbo = CommonDatautils.findDistrictById(d['D']) +// if(dbo){ +// def udata = new BasicDBObject() +//// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//},0) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -1037,6 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(1).xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + d['G'] = dist.city + d['H'] = dist.region + d['I'] = dist.name + } +}) //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) @@ -1099,6 +1241,30 @@ def findThirdCommCount(cityid, sqlInstance){ +//ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d-> +// def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district") +// if(obj){ +// def count = findGuidedPrice(sqlInstance236, obj._id.toString()) +// if(count){ +// d['M'] = "存在" +// } +// d['N'] = obj.name_cat +// d['O'] = obj.estate_type2 +// } +// +//}) + +def findGuidedPrice(sqlInstance, id){ + def count = 0 + def sql = "select count(*) as count from guided_price_gj where commid='${id}'" + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 03d4cfcb7..1d774446a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202210//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202208 - def endDate = 202210 + def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202211 + def endDate = 202301 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -36,6 +36,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new ThirdPartyPriceBatch(), args); } @@ -45,7 +49,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 - def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -67,7 +71,9 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + + saveToPgSql(rows) } @@ -618,4 +624,32 @@ class ThirdPartyPriceBatch extends BaseBatch { } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_third_party_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(50000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index fd96c4d80..7f072ef8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -50,8 +50,25 @@ class TradeStatLog { // task.deleteDealData2() // task.statTableCount() // task.filterTrade() +// task.updateDealPriceType() + } + def updateDealPriceType(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def c : citys){ + DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it-> + if(it.total_price instanceof Double){ + index ++ + println index+" "+it.city+" "+it.total_price+" "+it._id.toString() + + def udata = new BasicDBObject() + udata.put("total_price", (int)it.total_price) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata)) + } + } + } } def statTableCount(){ @@ -73,15 +90,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys_92; + def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","11月总量","12月总量","23年1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-08-01" - def date = "2022-09-01" - def endDate = "2022-10-01" - def newDate = "2022-11-01" + def startDate = "2022-11-01" + def date = "2022-12-01" + def endDate = "2023-01-01" + def newDate = "2023-02-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy new file mode 100644 index 000000000..857b62715 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class TestData { + + static main(args) { + def task = new TestData() +// task.run() + task.update() + } + + def update(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩1.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + if(d['E'] && d['G']=="人工"){ + def q = new BasicDBObject() + def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + + q.put("lat2", dLat2) + q.put("lng2", dLng2) + + Long lng = d['E'].toString().toDouble() * 10000000000L + Long lat = d['F'].toString().toDouble() * 10000000000L + q.put("lng", lng) + q.put("lat", lat) + DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q)) + } + } + }); + } + + def run(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def loc = getLocation(dist.city, dist.region, dist.name) + if(loc){ + d['E'] = loc.x + d['F'] = loc.y + println d['C']+" "+loc.x+" "+loc.y + } + } + +// if(!d['E']){ +// def loc = getAmapLocation("龙岩市", d['B'], d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } + +// if(!d['E']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def loc = getAmapOldLocation(dist.city, dist.region, d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } +// +// } + + }); + } + + def getLocation(city, region, name){ + String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + loc = data.location + } + }) + return loc; + } + + def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } + + def getAmapOldLocation(city, region, name){ + String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy new file mode 100644 index 000000000..59d62cfea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -0,0 +1,622 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +class ExportPgsqlBaseEvaluation { + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "base" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlBaseEvaluation() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + citys = citys.reverse() + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +// +// //第三方网站权重表 +// exportWebsiteRate(); +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// //处理小区情况 +// exportCommunityStaging()//没cityid +// exportBigdataCommunity()// +// +// exportCommunityBuildType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(DateUtils.formatDateByPattern(new Date(), "yyyy-MM") +"-18", new_dists) //现在按updatetime导出 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_bds_ceft + data << r.gujia_bds_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_bds_wgt + data << r.gujia_bds_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_bds_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_drc_prc + data << r.gujia_bds_drc_prc_src + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(price_date, new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujiabds_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujiabds_avprc_dt_val='${price_date}' "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujiabds_avprc_dt_val + data << r.gujia_bds_comm_avprc + data << r.gujiabdscomm_hgst_prc + data << r.gujiabdscomm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bds_mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_bds_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_bds_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(publishDate, new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and data_udt_tm>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + println city +" "+allRows.size() + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + pgsqlInstance.eachRow(sql,{r-> + index = index+1 + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_bds_unitprc + data << r.gujia_bds_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_bds_wbt_floor + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy new file mode 100644 index 000000000..fc3dfabeb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -0,0 +1,443 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportPgsqlEvaluationResult { + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "t_app_p9pha_plda." + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlEvaluationResult() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + //计算结果相关 +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + +// exportCommunityBasePrice(curMonth) + exportCommunityBasePrice1(curMonth) + +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) +// +// exportInOk() +// +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + + } + + def exportAvmEvaluationGJNB(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_bds_yrmo = ${yearMonth} and urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_bds_yrmo + data << r.gujia_bds_comm_num + data << r.gujiabdsvalmnycommnum + data << r.gujia_prc_cvr_rate + data << r.gujiabdsmnldpccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujiabdssamp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujiabdsnervetncomnum + data << r.gujianervethrunclcpct + data << r.gujiabds_tst_case_num + data << r.gujiabds_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiabdsmdltstcasenum + data << r.gujiabdsmdltstcommnum + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiabdstrdpttcasenum + data << r.gujiabdstrdpttcommnum + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val = ${yearMonth} AND urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_avprc + data << r.gujia_bds_clc_tp + data << r.gujia_bds_rlb + data << r.gujiabds_to_mkprc_mny + data << r.mdl_dt + data << r.gujiabds_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_bds_floor_num + data << r.gujiabdstot_floor_num + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${month} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; //gujiabds_clc_yrmo_val=${month} AND + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) + } + + + def exportCommunityBasePrice1(yearMonth){ + + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + + def exportCommunityPdescribeGj(month){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where gujia_bds_mo_val=${month} AND urbn_id='${cityCode}'"; + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bds_bss_prc + data << r.gujiabds_cfdc_dgr_grd + data << r.gujiabdsrt3mlgcasenum + data << r.gujiabdslistgclwstprc + data << r.gujiabdslistgcaemdval + data << r.gujiabdslistgcase_avg + data << r.gujiabdslistgcemaxprc + data << r.gujia_bds_mo_val + data << r.gujiabdsavgwminvalavg + data << r.gujiabdsavgwmaxvalavg + data << r.gujia_bds_rel_err_val + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujiabds_lmth_bss_prc + data << r.gujiabds_wthr_mnl_val + data << r.gujiabds_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujiabdsnervetuntwprc + data << r.gujia_prc_src + data << r.gujiabdsabtjlcmbitval + data << r.gujiabdsabtjlgcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujiabds_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujiabdsidxnmo_moonmo + data << r.gujiabdsa1yiminmmonmo + data << r.gujiabdsidxnyr_moonmo + data << r.gujiabdsat1yspdgrscor + data << r.gujiabdsarnd1kscavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(month){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}"; + def allRows = [] + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_case_id + data << r.gujia_bds_crterm + data << r.data_crt_tm + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); +// allRows.clear() +// } + }); + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } + } + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "ok文件上传Ftp完成......" + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + //CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + //println tableName+" 上传Ftp完成......" + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy new file mode 100644 index 000000000..5a4068b7c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -0,0 +1,618 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql + +class ExportPgsqlRonghe { + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + + def citys = [] + + def database = "base." + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def linux_prefix = "/ccb_ronghe/${curDate}/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlRonghe() + task.run(args) + } + + def run(args) { + def new_dists = [] + citys = CcbCommonUtils.getRunningCitys(args) + //按天导(每天一个文件夹) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists) + if (curDate.toString().endsWith("10")) { + + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() + +// } + +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); + } + } + + + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + + def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("10")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + } + createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false) + } + + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + def exportEvaluateRate(){ + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + + createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildprojFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommlayoutpicFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommunityGJRH(){ + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_comm_fmnm + data << r.gujia_admndiv_id + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_dtl_adr + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_comm_tp_val + data << r.gujiawthr_new_pan_ind + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildstruGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_stc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false) + + } + + def exportBigdataCommfeeGj(){ + + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.co_id + data << r.gujia_eps_val + data << r.gujia_hs_use + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_comm_eps_tp + data << r.gujia_eps_cmnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhousetypeGJ(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_comm_hs_tp + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildyearGj(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhouseuseGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_hs_use + data << r.gujia_ladder_accnum + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataPositionGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_obj_tp + data << r.gujia_obj_id + data << r.gujia_lo_tp + data << r.gujia_sn + data << r.lgt + data << r.ltt + data << r.gujia_hignas_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataAliasFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.rel_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_pcsst + data << r.gujia_vld_dt + data << r.gujia_vrsn_dsc + data << r.gujia_wthr_prim_nm + data << r.gujia_dtsrc + data << r.gujia_dsc + data << r.gujia_als_lvl + data << r.gujia_adr_tp_ecd + data << r.gujia_wthr_adt_ind + data << r.gujia_wthr_dspl_ind + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildingFJW(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_bld_als_dsc + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.floor_num + data << r.gujia_unit_num + data << r.gujia_accnum_val + data << r.gujiawthrext_lift_ind + data << r.gujia_lift_num + data << r.gujiajiladderjiactval + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_vld_ind + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + } + + def exportBigdataCityPrice(queryDate){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false) + } + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } + } + def fileName = linux_prefix+"${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false) + } + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 90a46050c..e3b08e9e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -27,6 +27,8 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @@ -238,7 +240,7 @@ class BigdataCitypriceBatch extends BaseBatch{ if (sell_avg_price || rent_avg_price) { def row = [] row << cityCodeDataMap.get(dist.city) - row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); row << dist._id.toString() if (sell_avg_price) { @@ -362,7 +364,7 @@ class BigdataCitypriceBatch extends BaseBatch{ if (sell_avg_price || rent_avg_price) { def row = [] row << cityCodeDataMap.get(city) - row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + row << curRunDate//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); if(region){ row << regionCodeDataMap.district_code diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 770ea6914..d2ad41623 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -31,8 +31,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - - def sqlInstance = null + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") public static def batchsize = 1000 static main(args) { @@ -42,13 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - if(citys.size()==92){ - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - }else{ - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - } + def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index a4a366773..8215007e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -4,10 +4,12 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql import java.sql.ResultSet @@ -27,11 +29,16 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + + citys = CcbCommonUtils.getRunningCitys(args) + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH()// @@ -49,7 +56,28 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) + + + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 + exportCommunityBasePrice(lastMonth) + exportCommunityPdescribeGj(lastMonth) + exportGuideStatistics(lastMonth) + exportBigdataSupplierStatistics(lastMonth) + exportBigdataModelingBase(lastMonth) + + //2月8号 临时导出苏州一年的数据 +// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 + } + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -526,7 +554,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) - if(curDate.toString().endsWith("28")){ + if(curDate.toString().endsWith("10")){ if(new_dists){ def newstr = "('"+new_dists.join("','")+"')" def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + @@ -801,6 +829,75 @@ class ExportDistrictRongHe extends BaseBatch{ println "EVALUATE_RATE_FJW 上传Ftp完成......" } + def exportThirdPartyPrice2(price_date, new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10"; + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = null + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + }else{ + sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}'"; + } + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload('rh', fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" @@ -812,6 +909,393 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" } + def exportCommunityBasePrice(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'"; + }else{ + sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}'"; + } + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh", fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + + def exportCommunityPdescribeGj(month){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'"; + }else{ + sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where CITYID='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh", fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportGuideStatistics(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; + }else{ + sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.MODEL_TYPE + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMM_SUM_COUNT + data << r.COMM_COUNT + data << r.RATIO + data << r.TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "guide_statistics 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh", fileName, curDate) + println "guide_statistics 上传Ftp完成......" + } + + + def exportBigdataSupplierStatistics(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}' and year_month=${time}"; + }else{ + sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.DATA_SOURCE + data << r.THIRD_COMM_COUNT + data << r.MODEL_TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "bigdata_supplier_statistics 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh", fileName, curDate) + println "bigdata_supplier_statistics 上传Ftp完成......" + } + + def exportBigdataModelingBase(time){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat" + } + def headers = [] + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = null; + if(CcbCommonUtils.citys_92.contains(city)){ + sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}' and year_month=${time}"; + }else{ + sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}'"; + } + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITY_CODE + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.ESTIMATE_DEAL_COUNT + data << r.ESTIMATE_COMM_COUNT + data << r.DEAL_COUNT + data << r.DEAL_COMM_COUNT + data << r.SALE_COUNT + data << r.SALE_COMM_COUNT + data << r.MODEL_TYPE + data << r.TYPE + allRows << data + }); + } + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + + println "bigdata_modeling_base 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh", fileName, curDate) + println "bigdata_modeling_base 上传Ftp完成......" + } + + def exportBigdataCityPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_CITYPRICE 上传Ftp完成......" + } + + + def exportBigdataDistPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.DISTID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DISTPRICE 上传Ftp完成......" + } + + + def exportBigdataCommPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.COMMID + data << r.SALEPRICE?r.SALEPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.HOUSEUSE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.DATATYPE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMPRICE 上传Ftp完成......" + } + + + def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" @@ -823,7 +1307,132 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } - def exportOk(curDate){ + def exportBigdataForsale(publishDate, new_dists){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + if(city=="苏州"){ + continue + } + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE +// data << r.ATTACH //融合房源没有这个字段 + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 161f82f00..c274629b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) @@ -25,40 +26,43 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + exportInOk() // } + SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) - + System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); } @@ -66,9 +70,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommunityGJRH(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys){ @@ -133,9 +137,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -162,9 +166,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -190,9 +194,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -228,9 +232,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ }); } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -258,9 +262,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -291,9 +295,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -323,9 +327,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -358,9 +362,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -382,9 +386,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ index++ def curStartStr = curStartDateStr - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -468,7 +472,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ stop = true } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + CcbFtpTestUtils.ftpUpload("ccb_ronghe", fileName, curDate) println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr @@ -482,9 +486,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] @@ -511,8 +515,13 @@ class ExportDistrictRongHeTest extends BaseBatch{ // } // } - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ ";// - allRows.addAll(createDeal(sql)) + for(def city : citys) { + if(CcbCommonUtils.citys_252.contains(city)){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}'"; + allRows.addAll(createDeal(sql)) + } + } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -675,9 +684,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); println "BIGDATA_DEAL_FJW 导出完成......" @@ -686,9 +695,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys) { @@ -743,14 +752,16 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -789,14 +800,14 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -836,14 +847,14 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -899,9 +910,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -910,9 +921,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -921,23 +932,35 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } - def exportOk(curDate){ -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } } - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) + def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "融合统计文件长度ok文件 上传Ftp完成......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy index 0e94c9551..07d453ef7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy @@ -16,7 +16,7 @@ class SSHHelperTest { private Session session = null; private Channel channel = null; - private String sftpHost = "192.168.0.86"; + private String sftpHost = "192.168.0.45"; private int sftpPort = 17495; private String sftpUserName = "ccb"; private String sftpPassword = "ccb!@#"; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy new file mode 100644 index 000000000..012ddd921 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.chongqing +import com.sun.org.apache.xml.internal.security.utils.Base64; +import org.apache.commons.lang3.StringUtils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESedeKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.security.Key; + +class Des3 { + private final static String encoding = "UTF-8"; + + static main(args) { + def task = new Des3() + task.run() + } + def run(){ + try { + String key = "111036369260679051122113"; + String iv = "23235645";//IV length must be 8 bytes long + //加密 + String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv); + System.out.println("密文: " + encryptStr); + //解密 + String decryptStr = decode(encryptStr, key, iv); + System.out.println("明文: " + decryptStr); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 方法描述:3DES加密 + * @author hyang + * + * @param plainText 明文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 密文 + * @throws Exception + */ + public static String encode(String plainText, String secretKey, String iv) + throws Exception { + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding"); + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.ENCRYPT_MODE, deskey, ips); + byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding)); + return Base64.encode(encryptData); + } + + /** + * 方法描述: 3DES解密 + * @author hyang + * + * @param encryptText 密文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 明文 + * @throws Exception + */ + public static String decode(String encryptText, String secretKey, String iv) + throws Exception { + if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) { + return ""; + } + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致 + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.DECRYPT_MODE, deskey, ips); + byte[] decryptData = cipher.doFinal(Base64.decode(encryptText)); + return new String(decryptData, encoding).trim(); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy new file mode 100644 index 000000000..a9a798124 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.chongqing + +import com.fangjia.SecretUtils + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.SecretKeySpec + +class Des3Test { + //定义加密算法,有DES、DESede(即3DES)、Blowfish + private static final String Algorithm = "DESede"; + private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113"; + + + /** + * 加密方法 + * @param src 源数据的字节数组 + * @return + */ + public static byte[] encryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥 + Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类 + c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /** + * 解密函数 + * @param src 密文的字节数组 + * @return + */ + public static byte[] decryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); + Cipher c1 = Cipher.getInstance(Algorithm); + c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /* + * 根据字符串生成密钥字节数组 + * @param keyStr 密钥字符串 + * @return + * @throws UnsupportedEncodingException + */ + public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{ + byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0 + byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组 + + /* + * 执行数组拷贝 + * System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位) + */ + if(key.length > temp.length){ + //如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, temp.length); + }else{ + //如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, key.length); + } + return key; + } + + /** + * @param args + */ + public static void main(String[] args) { + String msg = "3DES加密解密案例"; + System.out.println("【加密前】:" + msg); + + //加密 +// byte[] secretArr = Des3Test.encryptMode(msg.getBytes()); +// System.out.println("【加密后】:" + new String(secretArr)); + + String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0" + + //解密 + byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8")); + System.out.println("【解密后】:" + new String(myMsgArr)); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy new file mode 100644 index 000000000..2b163d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -0,0 +1,703 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + +class HailanTest { + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new HailanTest(); + task.run(); + } + + def run(){ + def inputfile = "D:\\数据\\海南\\统计.xlsx" +// match(inputfile) +// addPrice(inputfile) +// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") +// addHouse(inputfile) +// stat(inputfile) + hailanStat(inputfile) + } + + def addHouse(inputfile){ + + def datas = [] + datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向", + "景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"] + def index = 0 + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['L']){ + index ++ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + println index +" "+ row['C']+" "+ rets.size() + if(rets){ + def lastRets = [] + for(def r : rets){ + def fjwPrice = row['M'] + if(fjwPrice){ + def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price) + if(precent>=10){ + continue; + } + } + if(row['A'] && row['A']=="别墅"){ + def inputAare = row['F'].toString().toDouble() + def minArea = inputAare + 50 + def maxArea = inputAare + 100 + if(r.areamaxArea){ + continue; + } + } + + lastRets << r + if(lastRets.size()>=3){ + break; + } + } + + if(lastRets){ + for(def lr : lastRets){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + data << row['E'] + data << row['F'] + data << row['G'] + if(dist){ + data << dist.address + }else{ + data << "" + } + if(dist){ + data << dist.name + }else{ + data << "" + } + + data << "" + data << "" + if(lr.height){ + if(lr.height>6){ + data << "是" + }else{ + data << "否" + } + }else{ + data << "" + } + data << lr.floor + data << lr.height + data << lr.fitment + data << "" + data << lr.direction + data << "" + data << lr.build_type + data << lr.area + data << dist.property_life +// def year = dist.create_year +// if(year){ +// data << getHouseYear(year.toInteger())//已使用年限 +// }else{ +// data << ""//已使用年限 +// } + data << getHouseYear(dist.create_year , lr.build_date, dist.build_date) + + data << getEvalutaionPrice(lr, dist)//评估价(单价) + data << lr.avg_price + data << getDealPrice(dist)//成交单价 + data << row['M'] + datas << data + } + } + } + } + }) + ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据") + } + + def getDealPrice(dist){ + def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100) + def prices = [] + if(json && json.code==200){ + def result = json.result + if(result){ + for(def r : result){ + prices << r.avgPrice + } + } + } + def dealAvgPrice = null + if(prices){ + dealAvgPrice = (prices.sum()/prices.size()).toInteger() + } + return dealAvgPrice; + } + + def getHouseYear(create_year , house_build_date, dist_build_date){ + def year = null + if(house_build_date){ + year = getYear(house_build_date) + + }else if(dist_build_date){ + year = getYear(dist_build_date) + }else if(create_year){ + year = create_year + } + if(year){ + year = year.toInteger() + def curYear = new Date().getYear()+1900 + def modYear = curYear - year + return modYear.toInteger().toString() + } + return null + + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } + + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + + + def addCommId(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['H']){ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + if(dist){ + row['L'] = dist._id.toString() + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['L'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['M'] = priceObj.avg_price + } + } + } + }) + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + String city = row['H']//RegionData.formatCity(row['B']) + if(city){ + def address = row['C'] +// def region = row['C'] + println "[match] city:"+city+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) + if(jsonObject != null && jsonObject.precise!=-1) { + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("name") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("id") + } + }else{ + println row['C'] + } + + } + }) + } + + def stat(path){ + ExcelUtils.modify(path, { row -> + if(row['L'] && !row['H']){ + def city = row['L'] + def region = row['M'] + def dists = null + if(city && region){ + dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"]) + }else{ + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"]) + } + def distLists = [] + def villaLists = [] + for(def d : dists){ + if(d.estate_type2=="别墅"){ + villaLists << d + }else{ + distLists << d + } + } + row['J'] = dists.size() + + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + row['H'] = haveHouseList + row['I'] = haveVillaList +// row['N'] = distLists.size() +// row['O'] = villaLists.size() + } + }) + } + + def hailanStat(path){ + def total = 0 + def distTotal = 0 + def villaTotal = 0 + def haveHouseDistTotal = 0 + def haveHouseVillaTotal = 0 + def buildStructureTotal = 0 +// ExcelUtils.modify(path, { row -> +// if (row['L'] && !row['K']) { + //"海口","三亚","三沙","儋州", + def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def city : citys) { + + def dists = null +// if (city && region) { +// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } else { + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } + def distLists = [] + def villaLists = [] + for (def d : dists) { + if (d.estate_type2 == "别墅") { + villaLists << d + } else { + distLists << d + } + } + total = total + dists.size() + +// def structureCount = 0 +// for(def d : dists){ +// if(d.build_structure){ +// structureCount ++ +// } +// } +// println structureCount +// row['K'] = structureCount +// buildStructureTotal = buildStructureTotal + structureCount +// + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + haveHouseDistTotal = haveHouseDistTotal + haveHouseList + haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList + distTotal = distTotal + distLists.size() + villaTotal = villaTotal + villaLists.size() +// println c +" "+total+" "+buildStructureTotal +// } + } +// }) + + println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal + + } + + def findHaveHouseDist(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(dist) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + + def findThirdPrice(dist){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("name", dist.name) + q.put("s_date", new BasicDBObject('$gte', 202201)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def prices = [] + if(dbcursor){ + for(def d : dbcursor){ + def set = d.keySet(); + Iterator it = set.iterator() + while(it.hasNext()){ + def key = it.next() + if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){ + def price = d[key] + if(price){ + prices << price + } + } + } + } + } + return prices; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy new file mode 100644 index 000000000..d1e0db444 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy @@ -0,0 +1,328 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.apache.lucene.queryparser.classic.QueryParser +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +class MingShenPocStat { + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + + static main(args) { + def task = new MingShenPocStat() +// task.run() + task.officeRun() + } + + def officeRun(){ + def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx" + def datas = [] + ExcelUtils.modify(path, { d -> + if(d['B']){ + + def office = CommonDatautils.findDistrictById(d['C']) + if(office){ + def alias = [] + alias << office.name + if(office.alias){ + def nameAlias = office.alias + for(def o : nameAlias){ + if(o){ + alias << o + } + } + } + def aliasStr = escapeExprSpecialWord(alias.join("','")) + def houses = findOfficeHouseCount(office.city, office.region, aliasStr) + if(houses){ + if(houses.size()>=2){ + houses = houses[0..1] + } + for(def h : houses){ + def obj = findOfficeAvgPrice(office) + def rets = [] + rets << d['A'] + rets << d['B'] + rets << '' + rets << h.region + rets << office.address + rets << h.area + rets << h.floor + rets << h.height + def level = office.bed_kind + rets << level + rets << h.fitment + def date = office.finish_building_date + if(date){ + rets << date.substring(0, 4) + }else{ + rets << "" + } + + rets << "" + rets << "" + def price = "" + if(obj && obj.avg_price){ + price = obj.avg_price + rets << obj.avg_price + }else{ + rets << "" + } + + if(price){ + rets << (price*h.area).toInteger() + } + datas << rets + } + } + }else{ + def rets = [] + rets << d['A'] + rets << d['B'] + datas << rets + } + + // def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"]) + + } + }) + + ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本") + } + + def run(){ + def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx" +// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1) +// +// ExcelUtils.modify(path, { row -> +// if(row['C']) { +// def total = 0 +// def pre = 0 +// for(def d : datas){ +// if(row['C']==d['A']){ +// total = d['B'] +// pre = d['H'] +// break +// } +// } +// if(total){ +// def c = (total*pre/100).toInteger() +// row['E'] = c +// row['F'] = pre+"%" +// } +// } +// }) + + def totalcount = findHouseTotal("大连") + def count = findHouseCount("大连", "height") + + println DataHandleUtils.getDoubleRound(2, count*100/totalcount) + +// ExcelUtils.multiThreadingModify(path, { row -> +// if(row['C']) { + //def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"]) +// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"]) +// def total = districts.size() +// if(total){ +// def house_dist_count = findHouseStat(districts) +// def pre = house_dist_count*100/total +// pre = DataHandleUtils.getDoubleRound(2, pre) +// println total+" "+house_dist_count+" "+pre +// } +// def count = findHouseCount(row['C']) +// row['D'] = count +// println row['C']+" "+count +// row['H'] = findDistrictSellStat(row['C']) +// row['G'] = getNewDist(row['C']) + +// def totalcount = findHouseTotal(row['C']) + +// def total = row['Z'] +// def count = findHouseCount(row['C'], "build_date") +// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total) +// println count +" "+row['Q'] +// def obj = getFieldPrecent(row['C']) +// if(obj){ +// def detail = obj.district_detail +// row['E'] = detail.alias.toDouble()*100 +// row['F'] = detail.address.toDouble()*100 +// row['G'] = detail.lng.toDouble()*100 +// row['H'] = detail.buildType.toDouble()*100 +// row['I'] = detail.outPics.toDouble()*100 +// row['J'] = detail.developer.toDouble()*100 +// row['Q'] = detail.buildDate.toDouble()*100 +// row['R'] = detail.buildDate.toDouble()*100 +// } +// } +// }); + } + + def getFieldPrecent(city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1)) + if(rets){ + return rets[0] + } + return null + } + + + def getNewDist(city){ + String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' " + def count = null + ElasticUtils.executeSqlQuery(sql,{ data-> + count = data.count + }) + return count; + } + + def findDistrictSellStat(def city){ + def q = new BasicDBObject() + q.put("city", city) + def count = district_sell_stat_coll.count(q) + return count + } + + + def findHouseStat(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(rMap) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + def findHouseTotal(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouseCount(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; + } + + def findOfficeAvgPrice(office){ + def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + if(rets){ + def obj = rets[0] + return obj + } + return null + + } + + def findOfficeHouseCount(city, region, names){ + def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + return rets + } + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy new file mode 100644 index 000000000..112de372f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -0,0 +1,459 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MingShenPocTest { + + def total = 5000 + static main(args) { + def task = new MingShenPocTest() + task.run() + } + def run(){ + def path = "D:\\数据\\民生\\poc测试.xlsx" + //9级标准化 + //structureV2(path)//针对海南poc地址数据此接口不合适使用 + //省市区拆分 +// ner(path)//城市,区域带市带区 +// match(path) +// addPrice(path) +// addLatLng(path) +// addAroundPrice(path) +// addAuthor(path) + addHouse(path) + } + + def structureV2(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def jsonObj = FangjiaClientUtils.addressStructure(null, row['A']) + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + row['B'] = jsonObj.result.province + } + + if(jsonObj.result.city){ + def cityStr = null + if(jsonObj.result.city=="市辖区"){ + cityStr = jsonObj.result.province + }else{ + cityStr = jsonObj.result.city + } + row['C'] = RegionData.formatCity(cityStr) + } + + if(jsonObj.result.district){ + row['D'] = jsonObj.result.district + } + } + } + }) + } + + def ner(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['A'] && !d['B']) { + def json = FangjiaClientUtils.addressStand(null, d['A']) + if(json && json.entities){ + def entities = json.entities + for(def m : entities){ + if(m.type=="prov"){ + d['B'] = m.word + } + if(m.type=="city"){ + def cityStr = m.word + if(cityStr){ + d['C'] = cityStr//RegionData.formatCity(cityStr) + } + } + if(m.type=="dist") { + if(d['C']){ + d['D'] = m.word //RegionData.formatRegion(d['C'], m.word) + } + } + } + } + } + }); + } + + //可以根据测试数据添加 推荐楼盘名 + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C'] && !row['E']){ + def city = RegionData.formatCity(row['C']) + def region = null + if(city){ + region = RegionData.formatRegion(city, row['D']) + } + def address = row['A'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property" + if(jsonObject != null && jsonObject.precise!=-1) { + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("propertyType") + row['I'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['I'] + if(commid && !row['J']){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def house = [:] + house.area = 100 + def price = getEvalutaionPrice(house, dist) + if(price){ + row['J'] = price + } +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + } + } + }) + } + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + def addLatLng(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) { + def json = BaiduMapUtils.searchByAddress(d['A']) + if(json && json.location){ + d['K'] = json.location.lng + d['L'] = json.location.lat + } + } + }) + } + + + def addAroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['C'] && d['K']){ + def city = RegionData.formatCity(d['C']) + def dist = new BasicDBObject() + dist.city = city + dist.lng = d['K'].toString().toDouble()*10000000000L + dist.lat = d['L'].toString().toDouble()*10000000000L + dist.estateType = "普通住宅"//d['H'] + if(dist.lat) { + List list = DbUtils.findAroundDistrictByRadius(dist, 1000) + if (list) { + list = sortByDistance(dist, list); + if (list && list.size() > 10) { + list = list[0..9] + } + if (list) { + def prices = [] + for (def s : list) { + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if (priceObj) { + prices << priceObj.avg_price + } + } + if (prices) { + def arround_price = (int) (prices.sum() / prices.size()) + d['J'] = arround_price + } + } + } + } + } + }); + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + + def addAuthor(def path){ + def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['A']){ + if(index < avg){ + row['M'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def addHouse(path){ + def datas = [] + ExcelUtils.multiThreadingModify(path, { row -> + def commId= row['E'] + if(commId){ + def dist = CommonDatautils.findDistrictById(commId) + if(dist){ + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + if(rets){ + def data = [] + for(def r : rets){ + data << r.city + data << r.region + data << r.district_name + data << r.floor + data << r.height + data << r.fitment + data << r.direction + data << r.build_type + data << r.area + data << r.avg_price + datas << data + } + } + } + } + }); + + ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据") + } + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy new file mode 100644 index 000000000..3d685a1a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.io.FileUtils + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.util.Base64; + +/** + * 3DES加密工具类 + * @author qzz + */ +public class ThreeDESUtils { + + /** + * 加解密统一编码方式 + */ + private final static String ENCODING = "utf-8"; + + /** + * 加解密方式 + */ + private final static String ALGORITHM = "DESede"; + + /** + *加密模式及填充方式 + */ + private final static String PATTERN = "DESede/ECB/pkcs5padding"; + + /** + * 3DES加密 + * + * @param plainText 普通文本 + * @param sK 秘钥(24位密码) + * @return + * @throws Exception + */ + public static String encode(String plainText,String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + // 用密匙初始化Cipher对象 + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + // 执行加密操作 + byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING)); + return Base64.getEncoder().encodeToString(encryptData); + } + + /** + * 3DES解密 + * + * @param encryptText 加密文本 + * @return + * @throws Exception + */ + public static String decode(String encryptText, String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + + // 用密匙初始化Cipher对象 + cipher.init(Cipher.DECRYPT_MODE, secretKey); + // 正式执行解密操作 + byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText)); + return new String(decryptData, ENCODING); + } + + public static void main(String[] args) throws Exception { + + def rets = [] + def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt") + def index = 0 + for(def d : datas){ + index++ + def row = [] + def str = decode(d,"111036369260679051122113") + println str + def id = str.substring(str.length()-4) + row << id + row << str.substring(0, str.indexOf(id)) + rets << row + } + ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)") + +// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row -> +// //加密 +// String str = row['A'] +// //解密 +// row['B'] = (decode(str,"111036369260679051122113")); +// }); + + } + + public static def readTxt(def file){ + def lines = FileUtils.readLines(new File(file)) + def ret = []; + for(line in lines) { + ret << line; + } + return ret; + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 49bacf4ee..c6d2373a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -755,7 +755,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","湖畔佳苑") +// q.put("name","万达华府一期") DBObject keys = new BasicDBObject(); if(fields){ @@ -903,6 +903,36 @@ class CommonDatautils { } return null; } + + + public static def findSeaweedById(String id, def filelds){ + if(!id){ + return null; + } + try{ + DBObject q = new BasicDBObject(); + q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); + + DBObject keys = new BasicDBObject(); + filelds.each{ + keys.put(it, 1); + } + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() + } + }catch(Exception e){ + + } + return null; + } public static def findDistrictBySid(String sid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 8d77112a6..dd6f1e2dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] + def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index a11d0ce17..1b604d376 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -18,7 +18,9 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern +//小区月度均价表(带涨跌幅控制) @BatchOwner(Watcher.GaoJuan) class DistrictSellStatBatch extends BaseBatch { @@ -58,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) @@ -77,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch { }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } + print city +" "+districts.size() def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 0722c4464..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -111,8 +111,8 @@ class FzDataResultStat { def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; def datas1 = [] def datas2 = [] - datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] - datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] + datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] + datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] def row_num = 0 for(def row : rows){ if(row_num>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 32c4d1f34..1dd114d8a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -420,8 +420,8 @@ class HouseExportBatch extends BaseBatch{ row << seawater.name row << seawater.url -// row << seawater.p_date - row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") + row << seawater.p_date +// row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") rets << row } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 0231b21c0..47a3b331f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId + +DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") +def rets = [] +for(def d : datas){ + def arrs = d.split("@") + rets << ["commId":arrs[0], "avg_price":arrs[4]] +} + +for(def r : rets){ + if(r.avg_price){ + def udata = new BasicDBObject() + udata.put("date", 202210) + udata.put("avg", r.avg_price.toInteger()) + DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) + }else{ + println r.commId +" 不存在" + } +} + + + + //List list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv")) //println "过滤前:"+list.size()+ " "+new Date().toLocaleString() //def ret = [] @@ -48,29 +75,29 @@ import org.bson.types.ObjectId // return location; //} -List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) -def map1 = [:] -for(def s : list1){ - def row = s.split("@\\|@") - def key = row[2]+"##"+row[3]+"##"+row[4] - map1.put(key, row[5]) -} - -List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) -def map2 = [:] -for(def s : list2) { - def row = s.split("@\\|@") - def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row[5]) -} -def rows = [] -map1.each {k,v-> - def row = [] - def keys = k.split("##") - def lastPrice = map2.get(k) - rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] -} -ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") +//List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) +//def map1 = [:] +//for(def s : list1){ +// def row = s.split("@\\|@") +// def key = row[2]+"##"+row[3]+"##"+row[4] +// map1.put(key, row[5]) +//} +// +//List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +//def map2 = [:] +//for(def s : list2) { +// def row = s.split("@\\|@") +// def key = row[2] + "##" + row[3] + "##" + row[4] +// map2.put(key, row[5]) +//} +//def rows = [] +//map1.each {k,v-> +// def row = [] +// def keys = k.split("##") +// def lastPrice = map2.get(k) +// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] +//} +//ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") //ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> // def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) // if(obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index bcbe886de..44e4dd54c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 577aac71d..7a531b0e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -51,6 +51,7 @@ class AddAvgpriceBatch { for(def disKey : res2){ def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) + //发重算均价命令 CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date()) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index 64a84167e..ccca5758b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -17,15 +17,15 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); - //task.run() - task.run1() + task.run() +// task.run1() } def run(){ // def citys = CommonDatautils.findAllCity(); - def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"] + def citys = ["苏州","无锡","南通","徐州","连云港","宿迁","盐城","镇江","青岛","南京","常州","淮安","泰州","扬州","上海"] def ret = [] // ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"] ret << ["城市","小区总量"] @@ -45,7 +45,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\福建省小区数据统计.xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\小区数据统计.xlsx",ret,"数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy new file mode 100644 index 000000000..4cc9588c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +class ShLongShangHangLocation { + static main(args) { + def task = new ShLongShangHangLocation() + task.run() + } + + def run(){ + def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"] + def map = [:] + for(def r : rets){ + def location = getLocation(r) + if(location){ + map.put(r, location) + } + } + def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K'] + ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row-> + if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){ + for(def i=0;i + if(d['A'] && d['T']=="百度人工"){ + def jsonObj = BaiDuApiUtils.getDetailByUid(d['U']) + if(jsonObj && jsonObj.result){ + def result = jsonObj.result + if(result){ + d['J'] = result.uid + d['K'] = RegionData.formatRegion(result.city, result.area) + d['L'] = result.name + d['M'] = result.address + d['N'] = result.detail_info.tag + d['O'] = result.location.lng + d['P'] = result.location.lat + println result + } + } + } + }) + } + def marge(){ + def file1 = "D:\\数据\\银联\\银联人工1221(灵芝).xlsx" + def datas1 = ExcelUtils.read(file1, 0, 1) + ExcelUtils.modify("D:\\数据\\银联\\数据500条(12-21).xlsx",{d-> + if(d['A']){ + for(def r : datas1){ + if(r['A'] == d['A']){ + List pois = GaoDeUtils.getAmapById2(r['F']); + if(pois && pois.size()==1){ + def amapPoi = pois[0] + d['J'] = amapPoi.id + d['K'] = amapPoi.adname + d['L'] = amapPoi.name + d['M'] = amapPoi.address + d['N'] = amapPoi.type + def location = amapPoi.location + if(location){ + def arrs = location.split(",") + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + d['O'] = lng2 + d['P'] = lat2 + } + } + println pois + } + } + } + }) + } + + def run(){ + ExcelUtils.modify("D:\\数据\\银联\\数据500条(合并人工).xlsx",{row-> + def score = row['Q'] + if(row['B'] && !row['L']){ + def baiduCode = getBaiduMapCode(row['E']); + def city = RegionData.formatCity(row['E']); + if(!baiduCode){ + baiduCode = getBaiduMapCode(city); + } + + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + row['I'] = newKeys + + if(baiduCode){ + def map = getSimpleAddress(city, row['C']) + def simpleAddr = map.addr + if(map.city!="市辖区" && map.city!=city){ + city = RegionData.formatCity(map.city); + } + def rets = getBiduSearchDatas(baiduCode , simpleAddr, row['B'], null) + rets = getBiduSearchDatas(baiduCode , null, row['B'], rets) + rets = getBiduSearchDatas(baiduCode , null, simpleAddr, rets) + def bdRes = searchMapBySimilar(city, row['F'], row['B'], simpleAddr, rets, newKeys); + if (bdRes) { + row['J'] = bdRes.uid + row['K'] = bdRes.region + row['L'] = bdRes.name + row['M'] = bdRes.old_addr + row['N'] = bdRes.type + row['O'] = bdRes.baidu_lng2 + row['P'] = bdRes.baidu_lat2 + row['Q'] = bdRes.score + if(bdRes.matchType){ + row['R'] = bdRes.matchType + } + } + println row['B'] +" "+ bdRes + } + + } + }) + } + + def matchFlag(){ + ExcelUtils.multiThreadingModify("D:\\数据\\银联\\数据500条(合并人工).xlsx",{row-> + if(row['B'] && row['B'].contains("麦加美董村餐厅")){ + def datas = findQccData(row['B']) + if(datas){ + def max_score = 0 + def last_obj = null + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + def city = RegionData.formatCity(row['E']); + for(def d : datas){ + + def map = getSimpleAddress(null, d.address) + def inputMap = getSimpleAddress(null, row['C']) + if(inputMap && inputMap.city){ + city = inputMap.city + } + if(map && city==map.city){ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + def addr_score = 0 + if (map.addr && inputMap.addr) { + if(map.addr.contains("与")){ + def arrs = map.addr.split("与") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, s1); + if(temp_addr_score > addr_score){ + addr_score = temp_addr_score + } + } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, map.addr); + } + } + + def total_score = name_score + addr_score + if(total_score > max_score){ + max_score = total_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + } + println row['B'] +" "+ d.name+" "+total_score + }else{ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + if(map.addr && inputMap.addr && (inputMap.addr.contains(map.addr) || map.addr.contains(inputMap.addr))){ + max_score = name_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + println row['B'] +" "+ d.name+" "+name_score + } + } + } + if(max_score && max_score > 0.4){ + row['AB'] = last_obj.name + row['AC'] = last_obj.address + row['AD'] = last_obj.flag + row['AE'] = max_score + } + } + } + }); + } + + def findQccData(keywords){ + def q = new BasicDBObject() +// q.put("search_terms", keywords) + if(keywords.contains("(*")){ + keywords = keywords.toString().substring(0, keywords.toString().indexOf("(*")) + } + q.put("search_terms", Pattern.compile("^.*${keywords}.*\$", Pattern.CASE_INSENSITIVE)) + def dbcursor = QCC_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << ["name":d.name, "address":d.taxAddress, "flag":d.regStatus] + } + return rets; + } + def getBiduSearchDatas(baiduCode , simpleAddr, nameKey, rets){ + def searchKey = "" + if(simpleAddr){ + searchKey = simpleAddr +" "+nameKey + }else{ + searchKey = nameKey + } + def datas = searchBaiDuMap(searchKey, baiduCode.toInteger()); + if(!rets){ + return datas; + }else{ + def ids = [] + for(def r : rets){ + if(!ids.contains(r.uid)){ + ids << r.uid + } + } + for(def d : datas){ + if(!ids.contains(d.uid)){ + rets << d + } + } + return rets; + } + return null; + } + + + def getSimpleAddress(city, address){ + def jsonObj = FangjiaClientUtils.addressStructure(city, address) + def old_addr = "" + def map = [:] + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + map.pro = jsonObj.result.province + } + + if(jsonObj.result.city){ + if(jsonObj.result.city=="市辖区"){ + map.city = jsonObj.result.province + }else{ + map.city = jsonObj.result.city + } + map.city = RegionData.formatCity(map.city) + } + + if(jsonObj.result.district){ + map.region = jsonObj.result.district + } + if(jsonObj.result.street){ + def street = jsonObj.result.street + if(street){ + old_addr = street + map.street = street.replace("交叉口","").replace("交叉","") + } + } + if(jsonObj.result.streetNumber){ + def street_number = jsonObj.result.streetNumber + if(street_number){ + old_addr = old_addr + street_number + } + } + } + map.addr = old_addr.replace("交叉口","").replace("交叉","") + return map; + } + + def filterName(pro,city, region, name){ + def newKey = name.replace("餐饮服务有限公司","").replace("控股有限公司","").replace("有限责任公司", "").replace("有限公司","").replace("餐饮","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + if(region && newKey.contains(region) ){ + newKey = newKey.replace(region, "") + } + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) + } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + if(newKey.contains("(") || newKey.contains("(")){ + if(newKey.contains("(")){ + def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); + if(subName.endsWith("街") || subName.endsWith("路") || subName.contains("*") || subName.endsWith("店")){ + newKey = newKey.replace("("+subName+")", ""); + } + } + newKey = newKey.replace("(","").replace(")","").replace("(","").replace(")","") + if(newKey.toString().endsWith("店")){ + newKey = newKey.substring(0, newKey.size()-1) + } + } + return newKey; + } + + def filterName2(pro,city, region, name){ + def newKey = name.replace("餐饮服务有限公司","").replace("控股有限公司","").replace("有限责任公司", "").replace("有限公司","").replace("餐饮","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + if(region && newKey.contains(region) ){ + newKey = newKey.replace(region, "") + } + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) + } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + newKey = newKey.replace("(","").replace(")","").replace("(","").replace(")","") + return newKey; + } + + def getBaiduMapCode(city){ + if(city){ + return baiduCityCodeMap[city] + } + return city + } + + static def searchBaiDuMap(String address, int cityCode) { + def rets = [] + try { + Thread.sleep(500) + // String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&&auth=Dae1RaNR1LK8xdJTxN8SQOTRa2AH%3DL7vuxLNVENTxRBt6BG6B6WJWwGvzBU%40yYxvkGcuVtvvhguVtvyheuVtcvY1SGpuLtB2QKAZzvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwBVDzvFosSE21wiKDv7uvhgMuzVVtvrMhuENLtSrrlrIKHf0wd0vyMO7FOUySy&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a"; + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=19&from=webmap&sug_forward=&auth=E8LCawOwRbEOx6JVTvCIBI6W7zSz5vSXuxLNVENTRVLtUDHUDUYKYyHxzBU%40yYxvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwDWBWvveh3uxtwiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3VP8JSrrlrIKHf0wd0vyMO7FOUS7F&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a&device_ratio=2&tn=B_NORMAL_MAP&nn=0&u_loc=13513133,3631861&ie=utf-8&t=${System.currentTimeMillis()}&newfrom=zhuzhan_webmap" + def content = new HttpRequest().get(url, null, "utf-8") + JSONObject json = JSONObject.fromObject(content) + if (json && json.content && json.content.size() > 0) { + for (def loc : json.content) { +// String name = formatName(address) + def x = loc.x + def y = loc.y + if(x){ + x = x/100 + } + if(y){ + y = y/100 + } + if(x && y){ + def locJson = BaiDuApiUtils.geoconv(x, y, 6) + if(locJson){ + def baiduCity = RegionData.parseCityInAddress(loc.area_name) + def bean = [:] + def lng2 = locJson.get("x") + def lat2 = locJson.get("y") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2) + def lng = gmap.get("lng") + def lat = gmap.get("lat") + bean.uid = loc.uid + bean.baidu_lng2 = lng2 + bean.baidu_lat2 = lat2 + bean.longitude = lng + bean.latitude = lat + bean.name = loc.name + if(loc.address_norm){ + bean.address = getAddressInfo(loc.address_norm, loc.addr); + } + bean.old_addr = loc.addr + + if(!baiduCity){ + baiduCity = RegionData.formatCity(loc.city_name) + } + bean.cityname = baiduCity + if(loc.city_name){ + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + } + + if(loc.std_tag){ + bean.type = loc.std_tag + } + bean.flag = loc.acc_flag + rets << bean; + } + } + } + } + } catch(Exception e) { + searchBaiDuMap(address, cityCode) + } + return rets; + } + + def static formatName(String name) { + if(name.indexOf("·") != -1) { + name = name.replaceAll("·", "") + } + + if(name.indexOf(".") != -1) { + name = name.replace(".", "") + } + if(name.indexOf("(") != -1) { + name = name.replace("(", "") + } + if(name.indexOf(")") != -1) { + name = name.replace(")", "") + } + if(name.indexOf("-")!=-1){ + name = name.replace("-","") + } + return name.replaceAll("\\s+", "").toLowerCase() + } + + + static def getAddressInfo(def s, def addr){ + def arrs = s.split("]") + def map = [:] + if(arrs && arrs.size()>=3){ + if(arrs[0] && arrs[0].contains("PROV")){ + map.prov = arrs[0].substring(1, arrs[0].indexOf("(")) + } + if(arrs[1] && arrs[1].contains("CITY")){ + map.city = arrs[1].substring(1, arrs[1].indexOf("(")) + } + if(arrs[2] && arrs[2].contains("AREA")){ + map.region = arrs[2].substring(1, arrs[2].indexOf("(")) + } + } + if(addr){ + addr = addr.replace("[]","") + } + if(addr && map){ + if(map.region && !addr.contains(map.region)){ + addr = map.region + addr + } + if(map.city && !addr.contains(map.city)){ + addr = map.city + addr + } + + if(map.prov && !addr.contains(map.prov)){ + addr = map.prov + addr + } + } + return addr; + } + + + + def searchMapBySimilar(def city, def region, def name, def inpAddr, def rets, newKey){ + def bo = null; + if(rets){ + def maxScore = 0; + if(!bo || !maxScore){ + for(def o :rets){ + def s = o.name +// if(o.name && o.name.contains("(") && !o.name.toString().startsWith("(")){ +// s = o.name.substring(0, o.name.indexOf("(")) +// } +// s = formatName(s) + def addr = o.address + def mapKey = filterName(null, o.cityname, o.region, o.name) + def mapKey1 = filterName2(null, o.cityname, o.region, o.name) + + if(mapKey){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); + def score1 = CosineSimilarAlgorithm.getSimilarityRatio(mapKey1, newKey); + if(score1>score){ + score = score1 + } + def addr_score = 0 + if(inpAddr) { + def baidu_addr_map = getSimpleAddress(o.cityname, o.old_addr) + if(!baidu_addr_map.addr){ + baidu_addr_map.addr = AddressUtil.formatAddress(o.cityname, o.old_addr) + } + if(!baidu_addr_map.addr){ + if(o.region && !region.contains(o.region)){ + continue; + } + if(city!=o.cityname){ + continue; + } + } + if (baidu_addr_map.addr) { + if(baidu_addr_map.addr.contains("与")){ + def arrs = baidu_addr_map.addr.split("与") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, s1); + if(temp_addr_score > addr_score && (inpAddr.contains(baidu_addr_map.addr) || baidu_addr_map.addr.contains(inpAddr))){ + addr_score = temp_addr_score + } + } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr_map.addr); + if(addr_score>0 && inpAddr && baidu_addr_map.street && !inpAddr.contains(baidu_addr_map.street) && !baidu_addr_map.street.contains(inpAddr)){ + addr_score = 0 + } + } + + } + } + + if(addr_score>0 && score==0){ + addr_score = addr_score/2 + } + def total_score = score + addr_score +// if(city==o.cityname){ + if(total_score > maxScore){ + maxScore = total_score; + o.matchType = "城市相同,名称相似" + o.city = o.cityname + bo = o + bo.score = maxScore + + if(addr_score>0.6){ + o.matchType = "城市相同,名称相似,地址有关" + } + } +// if(bo && score!=1){ +// if(inpAddr){ +// def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) +// if(baidu_addr){ +// def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); +// //if(addr_score==1){ +// // o.matchType = "城市相同,名称相似,地址有关" +// bo.score = bo.score + addr_score +// maxScore = bo.score; +// //} +// } +// } + +// for(def word : filters){ +// if(mapKeyName.toString().endsWith(word)){ +// mapKeyName = mapKeyName.replace(word, "") +// } +// if(newName.toString().endsWith(word)){ +// newName = newName.replace(word, ""); +// } +// } +// +// +// if(mapKeyName && newName && mapKeyName.length()>=2){ +// score = CosineSimilarAlgorithm.getSimilarityRatio(mapKeyName, newName); +// if(score > maxScore){ +// maxScore = score +// o.matchType = "名称相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// }else{ +// if(mapKeyName.length() == newName.length()){ +// def mapKeyNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(mapKeyName, " ") +// def newNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") +// if(mapKeyNameBY == newNameBY){ +// maxScore = 0.977; +// o.matchType = "拼音相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// } +// } +// } +// } +// } +// } + } + } + } + } + return bo; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 6d692218b..52f2ea980 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-10-01" + static def statDate = "2023-01-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index f990711be..7a1d976ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId class AnjukeDistrictMatch { static main(args) { def task = new AnjukeDistrictMatch(); - task.run(); +// task.run(); // task.run1() +// task.updateField() + task.fenQiRun() } - def run1(){ - def regionPointsMap = findRegionRange("重庆") - def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + def fenQiRun(){ + def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"]) + def distMap = findFiqiDatas(dists) + println c +" "+dists.size() +" "+ distMap.size() + addFieldValue(distMap); + } } + + def addFieldValue(distMap){ + def index = 0 + distMap.each {k,v-> + if(v.master_district_name!="master"){ + def masterKey = v.city +"##"+v.region+"##"+v.master_district_name; +// def masterObj = distMap.get(masterKey) + def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district"); + if(masterObj){ + def udata = new BasicDBObject() + if(!v.virescence_percent && masterObj.virescence_percent){ + udata.put("virescence_percent", masterObj.virescence_percent) + } + + if(!v.build_area && masterObj.build_area){ + udata.put("build_area", masterObj.build_area) + } + + if(!v.all_area && masterObj.all_area){ + udata.put("all_area", masterObj.all_area) + } + + if(!v.cubage_percent && masterObj.cubage_percent){ + udata.put("cubage_percent", masterObj.cubage_percent) + } + + if(udata){ + index ++ + udata.put("m_date",new Date()) + def q = new BasicDBObject() + q.put("_id", new ObjectId(v._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + + } + } + } + println index; + } + + def findFiqiDatas(dists){ + def distMap = [:] + for(def d : dists){ + def master_district_name = d.master_district_name + if(master_district_name && master_district_name!="null"){ + distMap.put(d.city+"##"+d.region+"##"+d.name, d); + } + } + return distMap; + } + + + def updateField(){ + def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx" + ExcelUtils.multiThreadingModify(filePath, { row -> + if(row['Q'] && !row['Q'].contains("接口")){ + def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"]) + if(dist && !dist.virescence_percent){ + def val = row['K'] + if(!val){ + val = row['J'] + } + if(!val){ + val = row['M'] + } + if(!val){ + val = row['O'] + } + if(!val){ + val = row['G'] + } + if(!val){ + val = row['E'] + } + if(!val){ + val = row['F'] + } + if(!val){ + val = row['H'] + } + if(!val){ + val = row['I'] + } + if(!val){ + val = row['L'] + } + if(!val){ + val = row['N'] + } + if(!val){ + val = row['O'] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("virescence_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + row['U'] = 1 + } + } + } + }) + } + +// def run1(){ +// def regionPointsMap = findRegionRange("重庆") +// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) +// } def run(){ - - - def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx" + def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx" ExcelUtils.multiThreadingModify(filePath,{row-> - if(row['A'] && !row['J']){ + if(row['A'] && !row['Q']){ def city = row['B'] def region = row['C'] def name = row['D'] - def address = row['E'] -// def flag = "别名匹配" - -// def dist = matchDist(city, null, name) -// if(!row['J'] && name.contains("(")){ -// name = name.replace("(${region})","").replace("(","").replace(")","") -// flag = "去括号匹配" -// dist = matchDist(city, region, name) -// } -// if(!dist){ -// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") -// flag = "映射别名匹配" -// } -// if(!dist) { -// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") -// if(rets && rets.size()==1){ -// dist = rets[0] -// } -// } + def flag = "名称匹配" + def dist = matchDist(city, region, name) + if(!row['D'] && name.contains("(")){ + name = name.replace("(${region})","").replace("(","").replace(")","") + flag = "去括号匹配" + dist = matchDist(city, region, name) + } + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") + flag = "映射别名匹配" + } + if(!dist) { + def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()==1){ + dist = rets[0] + } + } // // if(!dist){ @@ -61,23 +182,21 @@ class AnjukeDistrictMatch { // } // } -// if(dist){ -// row['J'] = flag -// row['K'] = dist._id.toString() -// row['L'] = dist.region -// row['M'] = dist.name -// row['N'] = dist.address -// } - - def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) - if(jsonObject != null && jsonObject.precise!=-1) { -// row['J'] = "匹配接口" -// row['K'] = jsonObject.get("id") -// row['L'] = jsonObject.get("district") -// row['M'] = jsonObject.get("name") -// row['N'] = jsonObject.get("address") - row['M'] = jsonObject.get("name") + if(dist){ + row['Q'] = flag + row['R'] = dist._id.toString() + row['S'] = dist.region + row['T'] = dist.name + }else{ + def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['Q'] = "匹配接口" + row['R'] = jsonObject.get("id") + row['S'] = jsonObject.get("district") + row['T'] = jsonObject.get("name") + } } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ae1dee2e4..70249a1f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -73,7 +73,10 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8") @@ -90,10 +93,15 @@ class FangjiaClientUtils { def jsonObj = null def content = null try { - address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") - content = new HttpRequest().newGet(url, null, "utf-8") - jsonObj = JSONObject.fromObject(content) + if(address){ + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy new file mode 100644 index 000000000..c133d7283 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy @@ -0,0 +1,42 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +EsDataSource dataSource = new EsDataSource(); +String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + +findLogs() + +def findLogs(){ + def total = 0 + try{ + +// String sql = "select * from fangjia-log* where request like '/jnBank%'" +// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); + println houses.size() + def set = new HashSet() + def index = 0 + if(houses) { + for (def d : houses) { + def request = d.request + if(request && !d.domain.contains("k8s2.fangjia.com")){ + index ++ + println "[${index}] "+d.domain +" "+ request + } + } + } + + println set + + }catch(Exception e){ + e.printStackTrace() + } + + return total +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ce6242a31..6c4e5732c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, region) @@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { @@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "贝壳找房" - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def detailUrl = e.select(".title a").attr("href") + detail.url = detailUrl + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -398,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") -// if(e.text().contains("福州")){ + if(e.text().contains("重庆")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 8e74425cb..2bea3bcce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { @@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome { } detail.source = "链家在线" detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } + } } }catch(Exception ex){ @@ -470,11 +474,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["福州"] + def citys= ["重庆"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy new file mode 100644 index 000000000..3b8227240 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DongGuangApiTest { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new DongGuangApiTest() + task.run() + } + + def run(){ + // def rets = findEsData(); + def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv") + println "ES 加载完成" + def datas = evaluation(rets) + println "重估完成" +// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据") + FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true); + println "txt 生成" + exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas) + println "csv 生成" + } + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("#",-1) + if(row && row.size()>=2){ + def s = row[1] + JSONObject result = JSONObject.fromObject(s); + rows << result + } + + } + return rows; + } + + def evaluation(rets){ + def datas = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + def paramMap = map.parameter_map + rets << paramMap + if(rets.size()%10000==0){ + println "."+ new Date() + } + }) + return rets; + } + + + def exportCsv(path, list){ + File file = new File(path); + if (file.exists()){ + file.delete(); + } + file.append("用户输入城市,") + file.append("用户输入城市区域,") + file.append("用户输入城市地址,") + file.append("用户输入城市面积,") + file.append(System.getProperty("line.separator")); + list.each{ + def d = it; + file.append(d[0] + ",","gbk") + file.append(d[1] + ",","gbk") + file.append(d[2] + ",","gbk") + file.append(d[3] + ",","gbk") + if(d[4]){ + file.append(d[4]+",","gbk") + }else{ + file.append(",") + + } + + if(d[5]){ + file.append(d[5] +",","gbk") + }else{ + file.append(",") + + } + + + if(d[6]){ + file.append(d[6] +",","gbk") + }else{ + file.append(",") + + } + if(d[7]){ + file.append(d[7]+",","gbk") + }else{ + file.append(",") + } + file.append(System.getProperty("line.separator")); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index e1976f1cb..ca7c9e388 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -3,12 +3,15 @@ package com.fangjia.script.gj.test import ch.qos.logback.core.db.dialect.DBUtil import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo @@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1 def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) + +DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") +DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") +//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d -> +//}) + + +ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d -> + def info = d['A'] + if(info && !d['E']){ + def arrs = info.split("!~") + if(arrs && arrs[2] && arrs[3]){ +// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]); +// if(rMap) { +// d['E'] = rMap.name +// d['F'] = rMap.id +// } + + def address = null + if(d['G']){ + address = d['G'] + }else{ + address = arrs[4] + } + + def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + d['H'] = jsonObject.get("city") + d['I'] = jsonObject.get("district") + d['J'] = jsonObject.get("name") + d['K'] = jsonObject.get("id") + } + +// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4]) +// if(jsonObj && jsonObj.result) { +// def street = null +// def street_number = null +// if (jsonObj.result.street) { +// d['E'] = jsonObj.result.street +// } +// if (jsonObj.result.streetNumber) { +// d['F'] = jsonObj.result.streetNumber +// } +// d['G'] = jsonObj.result.name +// } + +// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3]) +// if(rets && rets.size()==1){ +// def obj = rets[0] +// d['B'] = obj.province +// d['C'] = obj.fjw_city +// d['D'] = obj.fjw_district +// } + } + } +}); +// +//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){ +// def q = new BasicDBObject() +// q.put("city_code", cityCode.toString().toInteger()) +// q.put("district_code", regionCode.toString().toInteger()) +// def dbcursor = FANGJIA_AREA_CODE.find(q) +// def rets = [] +// for(def d : dbcursor){ +// rets << d +// } +// return rets; +//} + //ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d-> // def city = RegionData.formatCity(d['E']) // if(city && !d['J']){ @@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi // } //}) -ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> - if(!d['D']){ - def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) - if(infos && infos.size()==1){ - def info = infos[0] - d['D'] = info.province_code - d['E'] = info.area_code - d['F'] = info.street_code - } - } -}) - - -def findTownshipInfo(city, region, coll){ - def q = new BasicDBObject() - q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) - q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) - - def dbcursor = coll.find(q) - def rows = [] - for(def d : dbcursor){ -// if(d.street_name.contains(region)){ - rows << d +//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> +// if(!d['D']){ +// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) +// if(infos && infos.size()==1){ +// def info = infos[0] +// d['D'] = info.province_code +// d['E'] = info.area_code +// d['F'] = info.street_code // } - } - return rows -} +// } +//}) +// +// +//def findTownshipInfo(city, region, coll){ +// def q = new BasicDBObject() +// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) +// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) +// +// def dbcursor = coll.find(q) +// def rows = [] +// for(def d : dbcursor){ +//// if(d.street_name.contains(region)){ +// rows << d +//// } +// } +// return rows +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index f43bda14e..3908e4ca6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -98,7 +98,7 @@ public class EmailConsumer extends BaseBatch { } } else { - println "send error ${errorCount}, sleep ${sleepTime / 1000}s ..." + println "${mail.receiver_email} send error ${errorCount}, sleep ${sleepTime / 1000}s ..." sleep(sleepTime); } } @@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); + email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com");//发送方 email.setSubject(mail.subject); //标题 @@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); + email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com"); email.setSubject(mail.subject); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 17b02bb6b..2a34dd4b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -247,6 +247,35 @@ class GaoDeUtils { return Collections.emptyList() } + + static List getAmapById2(id){ + def params = [:] + params.id = id + params.output = 'json' + params.key = "66e6771767ab293ace441553c36621c1" + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return Collections.emptyList() + } + /** * 获取详情 */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 78d3fceac..62e2524e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "链家"// - def citys = findCrawlCity(source, 202209); + def source = "365淘房"// + def citys = findCrawlCity(source, 202211); for(def city : citys){ - def count1 = findCount(source, city, 202209); - def count2 = findCount(source, city, 202210); + def count1 = findCount(source, city, 202211); + def count2 = findCount(source, city, 202212); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index f49a13675..dce00a5fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" - def esSources = ["链家":"链家"] + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 22eedff7f..2d9309dae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202209); - def count2 = findTotalCount(source, 202210); + def count1 = findTotalCount(source, 202212); + def count2 = findTotalCount(source, 202301); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy index 3bfe575e9..2eb3fcfed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.util +import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.RandomUtil import com.fangjia.script.addrepo.InvokeHttpRequest; import groovy.json.JsonSlurper @@ -103,24 +104,20 @@ class BaiduMapUtils { int recount = 0; while(!stop && recount<3) { try{ - def ret = ABuYunUtil.doGet(url, null, null, "utf-8"); - if (ret) { - JSONObject jsonObj = JSONObject.fromObject(ret); - - if(jsonObj.getInt("status") == 121){ - println "key over" - changeAK() - recount++; + HttpRequest request = new HttpRequest(); + try{ + String result = request.get(url,null,"UTF-8"); + JSONObject json = JSONObject.fromObject(result); + if(json.getInt("status") >= 300){ + changeAK(); + json = searchByAddress(address); }else{ - def result = jsonObj.result; - if(result){ - stop = true - return result; - }else{ - stop = true - } + return json.result } + }catch (Exception e){ + e.printStackTrace(); } + return null; }catch (Exception e){ recount++; println e.getMessage() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 7e6e73263..96709e230 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -335,8 +335,15 @@ public class DbUtils{ keys.append("lng2", 1) keys.append("estate_type2", 1) DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ - districts << dbo + if(district.estateType){ + if(dbo.estate_type2 == district.estateType){ + districts << dbo + } + }else{ + districts << dbo + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index c30a1353a..242e1b8a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -221,6 +221,9 @@ class ElasticUtils { searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } + if(results.size()%100000==0){ + println "."+ new Date() + } SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy index 34186b830..ec6fa0a0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy @@ -65,6 +65,8 @@ class MongoUtils { if (domainUrl == MongoDB_Domain_mongo) { port == 28017; } + + if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) { return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 7c37cae97..0d0647082 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -1105,6 +1105,30 @@ public class OpenApiClient { return null } + public static JSONObject addressEvaluation(city, region, address, size){ + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("size", String.valueOf(size)); + params.put("token", TOKEN); + String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8"); + JSONObject result = JSONObject.fromObject(response); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("地址估价失败:${city} ${address} ${size} ${result}") + } + } catch (Exception e) { + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateDistrictPriceByAddress(String city, String address, double size) { try { HttpRequest request = new HttpRequest(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 09877e434..c6dcfa9bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1 +1,18 @@ package com.fangjia.script.wenyan + +import com.fangjia.core.ProvinceData +import com.fangjia.script.util.Jms +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.client.MongoCollection +import org.bson.Document + +def jms = Jms.create("tcp://jms.db.fangjia.com:61616") + +jms.sendMail("1142375881@qq.com", "测试", "测试内容") +println('1111') \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy new file mode 100644 index 000000000..520cf9f99 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -0,0 +1,43 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + OPEN_URI_LOG.find(and(eq("user_id", "1007713"), eq("request_uri", "/property/evaluate"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) +// if (!hashSet.contains(result)) { + def item = [doc._id.toString(), "东莞银行总行", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item +// hashSet << result +// } else { +// println("已存在") +// } + } + if (count % 7 == 0) { + hashSet.clear() + } + + startDate = nextDate +} +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 8bca701d7..04e45fb88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -25,7 +25,7 @@ while (startDate.before(endDate)) { OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { @@ -38,4 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file +ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index fdcf818c3..4d301536b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" + def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" /** * 经纬度转地址
    diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java index 25eb7040d..4fff15570 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsService.java @@ -17,4 +17,6 @@ public interface AverageStatisticsService { AverageStatistics findSecendAverageStatictics(String city,String region,String name,String category); Map findFirstAverageStatictics(String city,String region,String name,String category, Date sDate); + + AverageStatistics save(AverageStatistics averageStatistics); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java index 772d5594c..fef0b66b5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java @@ -198,8 +198,14 @@ public class AverageStatisticsServiceImpl implements AverageStatisticsService { } return dataMap; } - - + + @Override + public AverageStatistics save(AverageStatistics averageStatistics) { + averageStatisticsDao.save(averageStatistics); + return averageStatistics; + } + + private boolean isValid(Integer value) { if (value != null && value > 1000) { return true; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java index e0e763666..d849f26c6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/UserServiceImpl.java @@ -7,12 +7,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fangjia.core.util.CacheUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import com.fangjia.dal.QueryParam; import com.fangjia.dal.sql.dao.UserDao; import com.fangjia.dal.sql.model.User; +import org.sothis.core.util.Closure; public class UserServiceImpl implements UserService { private UserDao userDao; @@ -28,10 +30,15 @@ public class UserServiceImpl implements UserService { return Collections.emptyList(); } - public User findByUserName(String userName) { - List params = new ArrayList(1); - params.add(new QueryParam("userName", userName)); - return userDao.findFirstByParams(params); + public User findByUserName(final String userName) { + return CacheUtils.getValue(UserServiceImpl.class, "findByUserName", userName, CacheUtils.CachePolicyEnum.TODAY, new Closure(){ + @Override + public User execute(String input) { + List params = new ArrayList(1); + params.add(new QueryParam("userName", userName)); + return userDao.findFirstByParams(params); + } + }); } public void updateUser(User user) {