diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java index 37e2d93f7..3a51dcc61 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java @@ -362,8 +362,8 @@ public class YugudanController { if (model.getId()>0) { //FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(model.getId()); Map data = new HashMap(); - if(StringUtils.isNotBlank(model.getAddress())){ - data.put("address", model.getAddress()); + if(StringUtils.isNotBlank(model.getCertificateAddress())){ + data.put("certificateAddress", model.getCertificateAddress()); } // if(model.getPeopleEditDate()!=null || model.getPeopleEditPrice()!=null){ // data.put("peopleEdit", 1); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java index d710efeed..10db294b2 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java @@ -150,8 +150,10 @@ public class YugudanModel extends BaseModel{ private String reportNum; private Integer askPriceId;//È˹¤Ñ¯¼Ûid - - + + private String certificateAddress; + + public int getId() { return id; } @@ -861,7 +863,12 @@ public class YugudanModel extends BaseModel{ public void setUnitNum(String unitNum) { this.unitNum = unitNum; } - - - + + public String getCertificateAddress() { + return certificateAddress; + } + + public void setCertificateAddress(String certificateAddress) { + this.certificateAddress = certificateAddress; + } } diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl index 53fd8f659..0d7cdf0e7 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl @@ -241,10 +241,10 @@ - 物业地å�€ + 产è¯�地å�€ - ${(evaluation.address)!} - + <#if evaluation?? && evaluation.certificateAddress??>${(evaluation.certificateAddress)!}<#else>${(evaluation.address)!} + diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl index 26c02dc4c..c2fa9b4a8 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl @@ -68,7 +68,7 @@ 城市 区域 楼盘å��ç§° - 估价地å�€ + 产è¯�地å�€ å¸�å�· é�¢ç§¯ 总价 @@ -83,7 +83,7 @@ ${it.city} ${it.region!} ${it.districtName!} - ${it.address!}<#if it.houseNum??>${it.houseNum!}å�· + <#if it.certificateAddress??>${(it.certificateAddress)!}<#else>${(it.address)!}<#if it.houseNum??>${it.houseNum!}å�· <#if it.unitNum?? && it.unitNum?length gt 1>${it.unitNum!}å�•å…ƒ <#if it.buildType!=3 && it.buildType!=8 && it.buildType!=0> <#if it.floorDesc?? && it.floorDesc?length gt 1>${it.floorDesc!}层 diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index d4cc2d5da..e11cd2246 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1341,6 +1341,7 @@ public class DataUtil { GZIPOutputStream outputStream = new GZIPOutputStream(bao); outputStream.write(content.getBytes("UTF-8")); outputStream.finish(); + System.out.println(bao.toString()); return bao.toByteArray(); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 12cb8ca9b..ffb9f957f 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -83,6 +83,8 @@ public class Loudong implements MongoEntity { private Date modifyDate; /**»§Êý*/ private Integer households; + /**Ò»Ìݼ¸»§µÄ»§*/ + private Integer roomCount; /** ×ܽ¨ÖþÃæ»ý */ private Double totalAreas; /** ¿ªÅÌʱ¼ä */ diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java index 8283df958..41941514a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java @@ -102,6 +102,7 @@ public class FjtEvaluation implements SqlEntity{ private String finishDateStr;//¿¢¹¤Ê±¼ä private String useType;//·¿ÎÝÓÃ; + private String certificateAddress; //²úÖ¤µØÖ· private Date downPdfDate;//ÏÂÔØ±¨¸æÊ±¼ä @@ -753,6 +754,16 @@ public class FjtEvaluation implements SqlEntity{ } + @Column(name = "certificate_address") + public String getCertificateAddress() { + return certificateAddress; + } + + public void setCertificateAddress(String certificateAddress) { + this.certificateAddress = certificateAddress; + } + + @Column(name = "down_pdf_date") public Date getDownPdfDate() { return downPdfDate; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java index ab916308a..54c19713d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java @@ -233,8 +233,8 @@ public class DealPriceController { Integer minPrice = null; Integer maxPrice = null; if(bo.getDistrictStat()!=null){ - minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); - maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1); } Date startDate = null; Date endDate = null; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 50b2700a2..ff5708174 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -434,8 +434,8 @@ public class HouseController { Integer minPrice = null; Integer maxPrice = null; if(bo.getDistrictStat()!=null){ - minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); - maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1); } List names = new ArrayList(); names.add(district.getName()); @@ -452,6 +452,62 @@ public class HouseController { return mav; } + + + public Object arroundlistAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("Â¥Å̱àºÅ»ò³ÇÊÐ,ÇøÓò£¬Ãû³ÆÎª±ØÌîÏî")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + if(district!=null){ + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("¸ÃÐ¡Çø²»´æÔÚ")); + return mav; + } + + //ÖܱßÐ¡Çø£¬Ä¬ÈÏ10 + List districts = districtService.findNearestDistrict(district, 1000, model.getDistCount()); + List houses = new ArrayList(); + for(District dist : districts){ + districtKey = new DistrictKey(dist.getCity(), dist.getRegion(), dist.getName()); + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List arroundhouses = seawaterDao.findSeawatersFromEs("sell", dist.getCity(), dist.getRegion(), dist.getName(), dist.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + if(arroundhouses!=null && arroundhouses.size()>0){ + houses.addAll(arroundhouses); + } + } + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + String getPicUrl(String url, List urls) { if (!urls.contains(url)) { urls.add(url); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java index 8f1311061..64fb38772 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java @@ -48,6 +48,7 @@ public abstract class SearchModel extends FangjiaApiModel { private Double area; protected Integer dataPeriod;//²éѯÌìÊý + private Integer distCount; /** * »ñÈ¡·¿Ô´ÊýÁ¿ÀàÐÍ sell:ֻȡµÃ¶þÊÖ·¿ÊýÁ¿ rent:ֻȡµÃ³ö×â·¿ÊýÁ¿ ÆäËû:²»»ñÈ¡ÊýÁ¿ @@ -376,4 +377,12 @@ public abstract class SearchModel extends FangjiaApiModel { public void setEndTime(Integer endTime) { this.endTime = endTime; } + + public Integer getDistCount() { + return (null != distCount && 0 != distCount) ? distCount : 20; + } + + public void setDistCount(Integer distCount) { + this.distCount = distCount; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index b59fe435b..1000b157c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -41,8 +41,15 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ rets.addAll(rows) println city +" "+rows.size() } + + def fileName = "/home/goojia/ccb/BIGDATA_COMMBUILDTYPE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat" + } + + def headers = []//["³ÇÊÐID","Ð¡ÇøID"," ½¨ÖþÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","Êý¾Ýʱ¼ä"] - FileUtils.batchCreateFile(rets, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); println "txt Éú³É" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 6cd17549e..b54b97e70 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -44,9 +44,7 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ -// if(["ËÕÖÝ","±±¾©","¹ãÖÝ","º¼ÖÝ","ºÏ·Ê","ÉîÛÚ","ʯ¼Òׯ","Î人","Î÷°²","ÖØÇì"].contains(city)){ -// continue -// } + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" Ð¡ÇøÁ¿£º"+dists.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d34dedcc2..f606cd13f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -52,48 +52,51 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) - def citys = CcbCommonUtils.citys; + def citys = CommonDatautils.findAllCity()//CcbCommonUtils.citys; + def ccbCitys = CcbCommonUtils.citys; -// for(def city : citys){ -// addDealPriceMatch(city); -// } -// println citys.unique().size() - def rows = [] - def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] -// if(["¹óÑô","·ðɽ","º¼ÖÝ","ÉòÑô","Ìì½ò","ÖÐɽ","ÐìÖÝ","ÉÜÐË","ÄÏͨ","¹þ¶û±õ","»ÝÖÝ","³¤É³","Çൺ","³¤´º","ÑĮ̀","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","Äþ²¨","³£ÖÝ","´óÁ¬","¼ÎÐË","½ð»ª","À¥Ã÷"].contains(city)){ -// ppeDists = getPpeGTE15Datas(city) -// } - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="ÉϺ£"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() + if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); } - - - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" ³É½»¹²£º"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt Éú³É" +//// println citys.unique().size() +// def rows = [] +// def headers = []//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// for(def city : citys){ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +//// if(["¹óÑô","·ðɽ","º¼ÖÝ","ÉòÑô","Ìì½ò","ÖÐɽ","ÐìÖÝ","ÉÜÐË","ÄÏͨ","¹þ¶û±õ","»ÝÖÝ","³¤É³","Çൺ","³¤´º","ÑĮ̀","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","Äþ²¨","³£ÖÝ","´óÁ¬","¼ÎÐË","½ð»ª","À¥Ã÷"].contains(city)){ +//// ppeDists = getPpeGTE15Datas(city) +//// } +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="ÉϺ£"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// +// def trades = getCityData(city, statMap, ppeDists) +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println city +" ³É½»¹²£º"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt Éú³É" } @@ -570,8 +573,10 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20220101", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd")) def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) final CountDownLatch latch = new CountDownLatch(trades.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 0bc461cc5..724682101 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220518, 20220619); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //²âÊÔͨ¹ýÆ¥Åä½Ó¿ÚÅ䷿ԴЧ¹û // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 98fbc2d09..db3beb6ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -377,7 +378,8 @@ class CcbCommonUtils { keys.put(field, 1); } } - def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) +// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000) println "ԭʼ:"+dbsour.count() def rets = [] for(def d : dbsour){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 532a96c43..3ac7f9a28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -34,6 +34,10 @@ class StagingDistrictBatch extends BaseBatch{ println city +" ·ÖÆÚÁ¿£º"+rets.size() } + def fileName = "/home/goojia/ccb/COMMUNITY_STAGING_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\COMMUNITY_STAGING_GJ.dat" + } def results =[] def headers = []//["·ÖÆÚID","Ð¡ÇøID","Ð¡ÇøÖ÷ÆÚID","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] @@ -41,7 +45,7 @@ class StagingDistrictBatch extends BaseBatch{ results.addAll(rows) //ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "Êý¾Ý") - FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 9af443b40..5648796e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,11 +30,11 @@ import java.util.regex.*; //def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -47,6 +47,16 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business +//def districts = CommonDatautils.findAllFiledDistrict("district", "ÄÏÄþ") +//for(def d : districts){ +// println d.name +//} + +//def dists = CommonDatautils.findAllDistrictByCity("ÄÏÄþ", "district", ["city","region","name"]) +//for(def d : dists){ +// def o = CommonDatautils.findDistrictById(d._id.toString()) +// println o._id.toString() + " "+o.name +//} //def citys = CcbCommonUtils.citys; //def rows = [] @@ -236,16 +246,16 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // println d // } //},0) -def rows = findTradePpeData(sqlInstance236) -// rows.unique() -def map=findTradeDetail(sqlInstance236,null) -def datas=[] -map.each{key,row-> - if(!rows.contains(key)){ - datas< +// if(!rows.contains(key)){ +// datas< -// if(d['A']){ +ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ð¡Çø\\ɾ³ýÖØ¸´Ð¡Çø.xlsx",{d-> + if(d['A']){ // def region = d['B'] // def newRegion = RegionData.formatRegion(d['A'], region) // if(newRegion) { // def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") -// if(dist){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println d -// } + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dist._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d + } // } -// } -//},0) + } +},0) //def citys = CcbCommonUtils.citys //def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 98d755434..85e2c71b4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -16,6 +16,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import java.util.concurrent.CopyOnWriteArrayList @@ -27,9 +28,20 @@ class CcbDistrictToDB { // task.run() // task.run1() // task.meituanRun() - task.meituanSqrentRun() +// task.meituanSqrentRun() // task.updateDistrict() // task.updateDongguangRegion() +// task.houseDistToDB() + task.sendCalPrice() + } + + def sendCalPrice(){ + def dists = CommonDatautils.findAllDistrictByCity("¶«Ý¸", "district", ["city","region","name","estate_type2"]) + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-27") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-06-14") + for(def data : dists){ + addPrice(data, startDate, endDate) + } } def updateDongguangRegion(){ @@ -80,6 +92,123 @@ class CcbDistrictToDB { } } + + def houseDistToDB(){ + def index = 0 + def addIndex = 0 + def keys = new CopyOnWriteArrayList() + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\¶þÆÚ\\ÐÂÔöÐ¡Çø_2739_0610.xlsx", { d -> + index++ + if(d['I']){ + def city = d['A'] + def mapInfo = getStreetInfo(d['P']+","+d['Q']) + def newRegion = null + if(mapInfo){ + if(d['A']=="¶«Ý¸"){ + newRegion = mapInfo.township + }else{ + newRegion = mapInfo.district + } + } + + newRegion = RegionData.formatRegion(city, newRegion) + + if(newRegion && newRegion.contains(d['B'])){ + def name = nameFileter(d['I']) + def alias = [] + if(d['I']!=name){ + alias << d['I'] + } + if(d['C']!=d['I'] && !alias.contains(d['C'])){ + alias << d['C'] + } + + def estateType2 = "ÆÕͨסլ" + if(name){ + if(name.contains("±ðÊû")){ + estateType2 = "±ðÊû" + }else if(name.contains("ÉÌס")){ + estateType2 = "ÉÌס" + }else{ + def type = d['O'] + if(type){ + if(type.contains("±ðÊû")){ + estateType2 = "±ðÊû" + }else if(type.contains("ÉÌס")){ + estateType2 = "ÉÌס" + } + } + } + } + def formatAddress = null + if (d['N']) { + formatAddress = AddressUtil.formatAddress(d['A'], d['N']) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + + if (d['P']) { + glat = d['P'].toString().toDouble() * 10000000000L + glng = d['Q'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['P'].toString().toDouble(), d['Q'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def key = city + "##" + newRegion + "##" + name + if (name && !keys.contains(key)) { + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if (!dist) { + addIndex++ + keys << key + def data = new BasicDBObject(); + data.put("name_cat", 0) + data.put("status", 0) + data.put("cat", "district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region", newRegion) + data.put("name", name) + data.put("estate_type2", estateType2) + data.put("address", formatAddress) + if (glat) { + data.put("lat", glat) + data.put("lng", glng) + data.put("lat2", blat) + data.put("lng2", blng) + } + def markMap = [:] + markMap.put("data_source", "»úÆ÷Èë¿â") + data.put("mark_map", markMap) + index++ + DbUtils.seaweed.save(data); + println addIndex + " " + data +// def stop = false +// while (!stop) { +// if (data._id.toString()) { +// addPrice(data) +// stop = true; +// } +// } + }else{ + if(alias){ + def data = new BasicDBObject(); + data.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + } + } + } + }else{ + println d['A']+" "+d['B']+" "+d['C'] +" ¾­Î³¶ÈËùÔÚÇø£º"+newRegion + } + } + }); + } + def meituanSqrentRun(){ def index = 0 ExcelUtils.modify("D:\\Êý¾Ý\\ÃÀÍÅ\\ÃÀÍÅ´ýÈë¿â.xlsx", { d -> @@ -228,11 +357,8 @@ class CcbDistrictToDB { } - def addPrice(dist){ - def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01") - def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24") + def addPrice(dist, startDate, endDate){ def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) - for(def weekend : weekends){ CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) } @@ -521,4 +647,30 @@ class CcbDistrictToDB { return null } + + def getLocationInfo(def city, def address){ + def gloJson = GaoDeApiUtils.geo(city, address) + if(gloJson) { + def glocation = null + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + glocation = geocodes.get(0).location + } + return glocation + } + return null + } + + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + return addressComponent; + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy index afa2233b6..99e779ac6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -17,11 +17,11 @@ class DistrictMarge { //task.run() // task.repeatAddress() // task.removeDist() - task.updateAddress() -// task.addMappingAlias() +// task.updateAddress() + task.addMappingAlias() } def updateAddress(){ - ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\¸üÐÂÐ¡ÇøµØÖ·.xlsx", { d -> + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\¸üÐÂÐ¡ÇøµØÖ·_0608.xlsx", { d -> if (d['A']) { def region = d['B'].trim() def newRegion = RegionData.formatRegion(d['A'], region) @@ -41,7 +41,7 @@ class DistrictMarge { } def removeDist(){ - ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\ɾ³ýÐ¡Çø.xlsx", { d -> + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\ɾ³ýÐ¡ÇøÊý¾Ý_245_0608.xlsx", { d -> if (d['A']) { def region = d['B'] def newRegion = RegionData.formatRegion(d['A'], region) @@ -52,6 +52,8 @@ class DistrictMarge { udata.put("status", 1) udata.put("m_date", new Date()) DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + }else{ + println d['A']+" "+d['C']+" ²»´æÔÚ" } }else{ println d['A'] + " ¸ñʽ»¯Ê§°Ü "+d['B'] @@ -62,7 +64,7 @@ class DistrictMarge { def addMappingAlias(){ def index = 0 - ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\ÐÂÔöÓ³Éä±ðÃû.xlsx", { d -> + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\ÐÂÔöÓ³Éä±ðÃû_0608.xlsx", { d -> index ++ if (d['A']) { def region = d['B'].trim() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index d81974c8b..830424944 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -25,8 +25,9 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def new_dists = ["350604","350605","410307","410308","450181"] //°´Ììµ¼£¨Ã¿ÌìÒ»¸öÎļþ¼Ð£© - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd")) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH() @@ -459,7 +460,7 @@ class ExportDistrictRongHe extends BaseBatch{ } - def exportBigdataDealFJW2(curDate, dealdate){ + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") @@ -472,86 +473,107 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] if(curDate.toString().endsWith("28")){ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.DEALID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.STREET - data << r.STREETNO - data << r.COMMID - data << r.COMMNAME - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.UNITID - data << r.UNITNO - data << r.UNITNAME - data << r.HOUSEID - data << r.ADDRESS - data << r.BUILDYEAR - data << r.PROPTYPE - data << r.PROPYEARS - data << r.PERIODNUM - data << r.FLOORNUM - data << r.FLOORTYPE - data << r.FLOORCNT - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.BUILDAREA - data << r.LIVINGAREA - data << r.HOUSESTYLE - data << r.ROOMCNT - data << r.LIVINGROOMCNT - data << r.KITCHENCNT - data << r.BATHROOMCNT - data << r.HOUSEUSE - data << r.HOUSESTRU - data << r.HOUSEASPECT - data << r.FITMENT - data << r.EVALUATENO - data << r.DEALUNITPRICE - data << r.DEALTOTALPRICE - data << r.DECLAREPRICE - data << r.EVALUATEPRICE - data << r.CONTRACTNO - data << r.DEALDATE - data << r.DEALTYPE - data << r.PAYTYPE - data << r.DEALPERIOD - data << r.IFELEVATOR - data << r.COMPAREAGARDEN - data << r.COMPAREATERRACE - data << r.COMPAREABALCONY - data << r.COMPAREABAYWINDOW - data << r.COMPAREAATTIC - data << r.COMPAREAGARAGE - data << r.COMPAREABASEMENT - data << r.COMPAREAOTHER - data << r.DATASOURCE - data << r.AGENCYID - data << r.AGENCYNAME - data << r.REMARK - data << r.CREATETIME1 - data << r.UPDATETIME1 - data << r.SOURCECODE - data << r.WORKDATE - allRows << data - }); + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + "where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + + " distid not in ${newstr} ";//CITYID='${cityCode}' and + def rets1 = createDeal(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + " where distid in ${newstr}" + def rets2 = createDeal(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + allRows = createDeal(sql) + } } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_DEAL_FJW ÉÏ´«FtpÍê³É......${endDateStr}" } + def createDeal(sql){ + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + + return allRows; + } + def exportAllBigdataDealFJW(){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE= 0) { + + out.write(buffer, 0, n); + + } + + } catch (Exception e) { + + e.printStackTrace(); + + } + + return out.toByteArray(); + + } + + public static String uncompressToString(byte[] bytes, String encoding) { + + if (bytes == null || bytes.length == 0) { + + return null; + + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + ByteArrayInputStream ins = new ByteArrayInputStream(bytes); + + try { + + GZIPInputStream ungzip = new GZIPInputStream(ins); + + byte[] buffer = new byte[256]; + + int n; + + while ((n = ungzip.read(buffer)) >= 0) { + + out.write(buffer, 0, n); + + } + + return out.toString(encoding); + + } catch (Exception e) { + + e.printStackTrace(); + + } + + return null; + + } + + public static String uncompressToString(byte[] bytes) { + + return uncompressToString(bytes, GZIP_ENCODE_UTF_8); + + } + + /** + * ×Ö·û´®×ª»¯³ÉΪ16½øÖÆ×Ö·û´® + * @param s + * @return + */ + public static String strTo16(String s) { + String str = ""; + for (int i = 0; i < s.length(); i++) { + int ch = (int) s.charAt(i); + String s4 = Integer.toHexString(ch); + str = str + s4; + } + return str; + } + + public static void main(String[] args) throws IOException { + + String s = "123"; + println "result: "+strTo16(s) + + System.out.println("×Ö·û´®³¤¶È£º"+s.length()); + + println compress(s) + + System.out.println("½âѹºó£º"+uncompress(compress(s)).length); + + System.out.println("½âѹ×Ö·û´®ºó£º£º"+uncompressToString(compress(s)).length()); + + } + +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MeituanStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MeituanStat.groovy new file mode 100644 index 000000000..aff0e94e9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MeituanStat.groovy @@ -0,0 +1,429 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.JacksonUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.bytebuddy.description.type.TypeDescription +import org.apache.commons.lang.StringUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.GZIPInputStream + + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + + +class MeituanStat { + + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + + private static DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) + + def arroundColl = CrawalDBUtil.getArroundTempCol(); + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new MeituanStat() + task.run() +// task.run1() +// task.run2() + } + + def run(){ + def hexStr = "\\x1f\\x8b\\x08\\x00\\x17U\\xa4b\\x02\\xff342\\x06\\x00\\xd2cH\\x88\\x03\\x00\\x00\\x00" + println URLDecoder.decode(hexStr.replaceAll("\\\\x", "%"), StandardCharsets.UTF_8.name()) + + println URLDecoder.decode(hexStr.replaceAll("\\\\x", "%"), "utf-8"); +// def s = com.fangjia.core.util.DataUtil.zip("123") +// String str = new String(s, "utf-8"); +// def r = com.fangjia.core.util.DataUtil.unzips(s) +// println s +// println str +// println r +// def citys = ["±±¾©","ÉϺ£","±£¶¨","³É¶¼","¹ãÖÝ","º¼ÖÝ","ÄϾ©","ÉîÛÚ","Ìì½ò","Î人","Î÷°²","ÖØÇì","¼ªÁÖ","³£ÖÝ","´óÁ¬","¶«Ý¸","·ðɽ","¸£ÖÝ","¹óÑô","¹þ¶û±õ","ºÏ·Ê","¼ÃÄÏ","À¥Ã÷","ÄÏÄþ","Äþ²¨","Çൺ","ÏÃÃÅ","ÉòÑô","ʯ¼Òׯ","ËÕÖÝ","̫ԭ","ÎÂÖÝ","ÎÞÎý","³¤´º","³¤É³","Ö£ÖÝ","°°É½","°üÍ·","´óÇì","´óͬ","º£¿Ú","ºªµ¦","ºâÑô","ºôºÍºÆÌØ","ºþÖÝ","»´°²","»ÝÖÝ","¼ÃÄþ","¼ÎÐË","½­ÃÅ","½ð»ª","¾£ÖÝ","À¼ÖÝ","Á¬ÔƸÛ","ÁÙÒÊ","ÁøÖÝ","ÂåÑô","ÃàÑô","Äϲý","ÄÏͨ","ÄÏÑô","ÆÎÌï","ÇØ»Êµº","ȪÖÝ","ÈýÑÇ","ÉÇÍ·","ÉÜÐË","̨ÖÝ","Ì©ÖÝ","ÌÆÉ½","Íþº£","Ϋ·»","ÎÚ³ľÆë","Îߺþ","ÏåÑô","ÐìÖÝ","ÑĮ̀","ÑγÇ","ÑïÖÝ","Ò˲ý","Òø´¨","ÔÀÑô","Õ¿½­","Õò½­","ÖÐɽ","Ö麣","ÖêÖÞ","×Ͳ©"] +// def path = "D:\\Êý¾Ý\\ÃÀÍÅ\\ÃÀÍÅ88¸ö³ÇÊÐͳ¼Æ.xlsx" +// def rets =[] +// rets << ["³ÇÊÐ", "Â¥ÅÌ×ÜÁ¿"] +// for(def c : citys){ +// def count = CommonDatautils.findAllDistrictCount("district", c) +// rets << [c, count] +// } +// ExcelUtils.write(path, rets, "×ÜÁ¿") + } + + def run1(){ + def citys = ["±±¾©","ÉϺ£","±£¶¨","³É¶¼","¹ãÖÝ","º¼ÖÝ","ÄϾ©","ÉîÛÚ","Ìì½ò","Î人","Î÷°²","ÖØÇì","¼ªÁÖ","³£ÖÝ","´óÁ¬","¶«Ý¸","·ðɽ","¸£ÖÝ","¹óÑô","¹þ¶û±õ","ºÏ·Ê","¼ÃÄÏ","À¥Ã÷","ÄÏÄþ","Äþ²¨","Çൺ","ÏÃÃÅ","ÉòÑô","ʯ¼Òׯ","ËÕÖÝ","̫ԭ","ÎÂÖÝ","ÎÞÎý","³¤´º","³¤É³","Ö£ÖÝ","°°É½","°üÍ·","´óÇì","´óͬ","º£¿Ú","ºªµ¦","ºâÑô","ºôºÍºÆÌØ","ºþÖÝ","»´°²","»ÝÖÝ","¼ÃÄþ","¼ÎÐË","½­ÃÅ","½ð»ª","¾£ÖÝ","À¼ÖÝ","Á¬ÔƸÛ","ÁÙÒÊ","ÁøÖÝ","ÂåÑô","ÃàÑô","Äϲý","ÄÏͨ","ÄÏÑô","ÆÎÌï","ÇØ»Êµº","ȪÖÝ","ÈýÑÇ","ÉÇÍ·","ÉÜÐË","̨ÖÝ","Ì©ÖÝ","ÌÆÉ½","Íþº£","Ϋ·»","ÎÚ³ľÆë","Îߺþ","ÏåÑô","ÐìÖÝ","ÑĮ̀","ÑγÇ","ÑïÖÝ","Ò˲ý","Òø´¨","ÔÀÑô","Õ¿½­","Õò½­","ÖÐɽ","Ö麣","ÖêÖÞ","×Ͳ©"] + def path = "D:\\Êý¾Ý\\ÃÀÍÅ\\ÃÀÍÅ88¸ö³ÇÊÐͳ¼Æ.xlsx" + def rets =[] + rets << ["³ÇÊÐ","ID","ÇøÓò","Ð¡ÇøÃû","Ð¡Çø±ðÃû","µØÖ·","µØÖ·±ðÃû","ÎïÒµÀàÐÍ","½¨ÖþÀàÐÍ","¾­¶È","γ¶È","°å¿é","¿ª·¢ÉÌ","ÎïÒµ¹«Ë¾", + "»·Ïß","ÉÌȦ","Â¥¶°×ÜÊý","»§Êý","ÎïÒµ·Ñ","Õ¼µØÃæ»ý","½¨ÖþÃæ»ý","ÈÝ»ýÂÊ","ÂÌ»¯ÂÊ","Í£³µÎ»","Í£³µÎ»£¨ÃèÊö£©","²úȨÄêÏÞ", + "²úȨÐÔÖÊ","½¨³ÉÄê´ú","¿ª¹¤Ê±¼ä","¿ªÅÌʱ¼ä","¿¢¹¤Ê±¼ä","½»·¿Ê±¼ä","×°ÐÞÇé¿ö","½¨Öþ½á¹¹","ÖܱßÅäÌ×","½»Í¨×´¿ö","Ïà¹ØÐÅÏ¢", + "ÃèÊö","ÊÛÂ¥µØÖ·","ÊÛÂ¥µç»°","ÕÛ¿ÛÐÅÏ¢","Ô¤ÊÛÐí¿ÉÖ¤","·¿²úÆÀ¼¶","·¿²ú¾ù¼Û","×â½ð¾ù¼Û"] + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["lat","lng","loudong_count","mobile","car_site","fitment","relate_info","description","sell_address","discount","facilities","traffic","region","name"]) + def rMap = statFacilices(dists, c) + def obj = findFieldCov(c) + def rentCount = CommonDatautils.findSqmetreRents(c, DateUtils.parseFromTightByPattern("2022-05-31","yyyy-MM-dd")) + def block = obj.district_detail.block?obj.district_detail.block*100:'' + def alias = obj.district_detail.alias?obj.district_detail.alias*100:'' + def address= obj.district_detail.address?obj.district_detail.address*100:'' + def addrAlias = obj.district_detail.addrAlias?obj.district_detail.addrAlias*100:'' + def estateType2 = obj.district_detail.estateType2?obj.district_detail.estateType2*100:'' + def buildType = obj.district_detail.buildType?obj.district_detail.buildType*100:'' + def lng = obj.district_detail.lng?obj.district_detail.lng*100:'' + def developer = obj.district_detail.developer?obj.district_detail.developer*100:'' + def estateCompany = obj.district_detail.estateCompany?obj.district_detail.estateCompany*100:'' + def loopLineInfo = obj.district_detail.loopLineInfo?obj.district_detail.loopLineInfo*100:'' + def businessQuarter = obj.district_detail.businessQuarter?obj.district_detail.businessQuarter*100:'' + def loudong = '' + def loudongcount = getCount(dists,"loudong_count") + if(obj.existing_district_count && loudongcount){ + def pre = loudongcount*100/obj.existing_district_count + loudong = DataHandleUtils.getDoubleRound(2, pre) + } + + def householdCount = obj.district_detail.householdCount?obj.district_detail.householdCount*100:'' + def estateCharge = obj.district_detail.estateCharge?obj.district_detail.estateCharge*100:'' + def allArea = obj.district_detail.allArea?obj.district_detail.allArea*100:'' + def buildArea = obj.district_detail.buildArea?obj.district_detail.buildArea*100:'' + def cubagePercent = obj.district_detail.cubagePercent?obj.district_detail.cubagePercent*100:'' + def virescencePercent = obj.district_detail.virescencePercent?obj.district_detail.virescencePercent*100:'' + def carSite = obj.district_detail.carSite?obj.district_detail.carSite*100:'' + def carSiteStr = '' + def car_site_count = getCount(dists,"car_site") + if(obj.existing_district_count && car_site_count){ + def pre = car_site_count*100/obj.existing_district_count + carSiteStr = DataHandleUtils.getDoubleRound(2, pre) + } + def propertyLife = obj.district_detail.propertyLife?obj.district_detail.propertyLife*100:'' + def houseProperty = obj.district_detail.houseProperty?obj.district_detail.houseProperty*100:'' + def buildDate = obj.district_detail.buildDate?obj.district_detail.buildDate*100:'' + def startBuildingDate = obj.district_detail.startBuildingDate?obj.district_detail.startBuildingDate*100:'' + def openDate = obj.district_detail.openDate?obj.district_detail.openDate*100:'' + def finishBuildingDate = obj.district_detail.finishBuildingDate?obj.district_detail.finishBuildingDate*100:'' + def liveDate = obj.district_detail.liveDate?obj.district_detail.liveDate*100:'' + def fitment = '' + def fitment_count = getCount(dists,"fitment") + if(obj.existing_district_count && fitment_count){ + def pre = fitment_count*100/obj.existing_district_count + fitment = DataHandleUtils.getDoubleRound(2, pre) + } + def buildStructure = obj.district_detail.buildStructure?obj.district_detail.buildStructure*100:'' + def arround= '' + def facilities_count = getCount(dists,"facilities") + if(obj.existing_district_count && facilities_count){ + def pre = facilities_count*100/obj.existing_district_count + arround = DataHandleUtils.getDoubleRound(2, pre) + } + + def traffic = '' + def traffic_count = getCount(dists,"traffic") + if(obj.existing_district_count && traffic_count){ + def pre = traffic_count*100/obj.existing_district_count + traffic = DataHandleUtils.getDoubleRound(2, pre) + } + + def relateInfo = '' + def relate_info_count = getCount(dists,"relate_info") + if(obj.existing_district_count && relate_info_count){ + def pre = relate_info_count*100/obj.existing_district_count + relateInfo = DataHandleUtils.getDoubleRound(2, pre) + } + def description = '' + def description_count = getCount(dists,"description") + if(obj.existing_district_count && description_count){ + def pre = description_count*100/obj.existing_district_count + description = DataHandleUtils.getDoubleRound(2, pre) + } + def sellAddress = '' + def sell_address_count = getCount(dists,"sell_address") + if(obj.existing_district_count && sell_address_count){ + def pre = sell_address_count*100/obj.existing_district_count + sellAddress = DataHandleUtils.getDoubleRound(2, pre) + } + def mobile ='' + def mobile_count = getCount(dists,"mobile") + if(obj.existing_district_count && mobile_count){ + def pre = mobile_count*100/obj.existing_district_count + mobile = DataHandleUtils.getDoubleRound(2, pre) + } + def discount = '' + def dis_count = getCount(dists,"discount") + if(obj.existing_district_count && dis_count){ + def pre = dis_count*100/obj.existing_district_count + discount = DataHandleUtils.getDoubleRound(2, pre) + } + def licence = obj.district_detail.licence?obj.district_detail.licence*100:'' + def price = obj.price_coverage?obj.price_coverage*100:'' + def level = price + def rentPrie = '' + if(obj.existing_district_count && rentCount){ + def pre = rentCount*100/obj.existing_district_count + rentPrie = DataHandleUtils.getDoubleRound(2, pre) + } + + def db_fc = '' + def db_bus = '' + if(rMap){ + def count = obj.existing_district_count + if(rMap.bus){ + def pre = rMap.bus*100/count + db_bus = DataHandleUtils.getDoubleRound(2, pre) + } + if(rMap.fc){ + def pre = rMap.fc*100/count + db_fc = DataHandleUtils.getDoubleRound(2, pre) + } + } + rets << [c, 100,100,100,alias,address,addrAlias,estateType2,buildType,lng,lng,block,developer,estateCompany, + loopLineInfo,businessQuarter,loudong,householdCount,estateCharge,allArea,buildArea,cubagePercent,virescencePercent,carSite,carSiteStr,propertyLife, + houseProperty,buildDate,startBuildingDate,openDate,finishBuildingDate,liveDate,fitment,buildStructure,arround,traffic,relateInfo, + description,sellAddress,mobile,discount,licence,level,price,rentPrie, db_bus, db_fc] + println c +" Íê³É" + } + ExcelUtils.write(path, rets, "¸²¸ÇÂÊ") + } + + def run2(){ + ExcelUtils.modify("D:\\Êý¾Ý\\ÃÀÍÅ\\ÃÀÍÅ88¸ö³ÇÊÐͳ¼Æ.xlsx",{d-> + def dists = CommonDatautils.findAllDistrictByCity(d['A'], "district", ["name","lat","lng"]) + def rMap = statFacilices(dists) + if(rMap){ + def count = dists.size() + if(rMap.bus){ + def pre = rMap.bus*100/count + d['AT'] = DataHandleUtils.getDoubleRound(2, pre) + } + if(rMap.fc){ + def pre = rMap.fc*100/count + d['AU'] = DataHandleUtils.getDoubleRound(2, pre) + } + } + }) + } + + def findFieldCov(c){ + def q = new BasicDBObject() + q.put("city", c) + q.put("weekend", DateUtils.parseFromTightByPattern("2022-06-05", "yyyy-MM-dd")) + def obj = CITY_INFO_CONN.findOne(q) + return obj + } + + def getCount(dists,field){ + def count = 0 + for(def d : dists){ + def val = d.get(field) + if(val && !val.toString().contains("ÔÝÎÞ")){ + count++ + } + } + return count + } + + def statFacilices(dists){ +// def distMap = findArroundFaciltices(city) + def bus_count = 0 + def fc_count = 0 +// for(def d : dists){ +// def lng = d.lng/10000000000.0 +// def lat = d.lat/10000000000.0 +// def fcCount = findFacByEs(lng, lat, ["160100","060400","141200","141201","141202","141203","090100","160300"]) +// def busCount = findFacByEs(lng, lat, ["150700", "150500"]) +// println fcCount +// println busCount +// } + + def total = new AtomicLong() + def busCount = new AtomicLong() + def fcCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i0){ + busCount.addAndGet(1) + } + if(fc && fc>0){ + fcCount.addAndGet(1) + } + println total.get()+" "+ d.name + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + + def rMap = [:] + rMap.bus = busCount.get() + rMap.fc = fcCount.get() + return rMap + } + + def findFacByEs(lng, lat, codes){ + def must = [] + def term1 = ["terms":["typecode":codes]] + + must << term1 + def params = ["query":["bool":["must":must,"filter":["geo_distance": [ + "distance": "1km","location": ["lat": lat,"lon": lng]]]]]] + def s = httpSaveData(params, "http://192.168.0.206:9200/amap_poi/_search"); + def total = 0 + if(s){ + def doc = new groovy.json.JsonSlurper().parseText(s) + def hits = doc.hits + if(hits){ + total = hits.total.value + } + } + return total + } + + + + def findArroundFaciltices(city){ + def q = new BasicDBObject() + q.put("matched_city", city) + def dbcursor = arroundColl.find(q) + def distMap = [:] + for(def d : dbcursor){ + + def key = d.matched_region+"#"+d.matched_name + def dMap = distMap.get(key) + if(!dMap){ + dMap = [:] + distMap.put(d.region+"#"+d.district_name, dMap) + } + + def bus = 0 + def fc = 0 + if(d.type){ + if(d.type.contains("¹«½»") || d.type.contains("µØÌú")){ + bus = 1 + fc = 1 + }else{ + fc = 1 + } + } +// if((d.bus && d.bus.size()>0) || (d.school && d.school.size()>0) || (d.bank && d.bank.size()>0) ||(d.supermarket && d.supermarket.size()>0) || (d.hospital && d.hospital.size()>0) || (d.atm && d.atm.size()>0)|| (d.university && d.university.size()>0)|| (d.middle_school && d.middle_school.size()>0) || (d.primary_school && d.primary_school.size()>0)|| (d.kindergarten && d.kindergarten.size()>0)){ +// fc = 1 +// } + dMap.bus = bus + dMap.fc = fc + } + return distMap + } + + def findAllFaciltices(city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = on_line.find(q) + def distMap = [:] + for(def d : dbcursor){ + def bus = 0 + def fc = 0 + if(d.bus && d.bus.size()>0){ + bus = 1 + } + if((d.bus && d.bus.size()>0) || (d.school && d.school.size()>0) || (d.bank && d.bank.size()>0) ||(d.supermarket && d.supermarket.size()>0) || (d.hospital && d.hospital.size()>0) || (d.atm && d.atm.size()>0)|| (d.university && d.university.size()>0)|| (d.middle_school && d.middle_school.size()>0) || (d.primary_school && d.primary_school.size()>0)|| (d.kindergarten && d.kindergarten.size()>0)){ + fc = 1 + } + def dMap = [:] + dMap.bus = bus + dMap.fc = 1 + distMap.put(d.region+"#"+d.district_name, dMap) + } + return distMap + } + + + + def httpSaveData(def params, def url){ + def header = [:] + header.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz") + return doPost(url,header, params, true, "utf-8") + } + + + def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + StringBuilder buffer = new StringBuilder(); + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=" + CHARSET); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //Êä³öÁ÷ + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + + } + return buffer.toString(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index b8e3cd5a1..5abaef7c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3200,6 +3200,14 @@ class CommonDatautils { return dbo; } + static def findSqmetreRents(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def count = SQMETRE_RENT_COLL.count(q) + return count; + } + static def findNewSqmetreRent(def city, def region, def name){ def q = new BasicDBObject() q.put("city", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy index f3f386be6..5c1e8c152 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/FjtResponseTimeAnaly.groovy @@ -34,7 +34,8 @@ class FjtResponseTimeAnaly extends BaseBatch{ @Override public void process(Object args) { - run(); + //run(); + jms.sendMail("gaojuan@fangjia.com", "²âÊÔ·¢Óʼþ", "ÊÕµ½ÁË£¿") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7e5074c7c..563e6ab47 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //¸£ÖݵÚÈý·½¾ù¼Ûͳ¼Æ,·¢ËͶ¤¶¤ÏûÏ¢ //def datas = ExcelUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝ4ÔÂ2.xlsx",0,1) //def datas = CsvUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv") +def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202205.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -136,8 +136,8 @@ def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\Êý¾Ý\\¸£ÖÝ\\¶à·½¾ù¼Û¶Ô±Èͳ¼ÆÊý¾Ý"+curMonth+".xlsx",rets,"ͳ¼Æ") def s = "µÚÈý·½¾ù¼Û×ÜÁ¿£º${total}\n6Çø×ÜÁ¿£º${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s -def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -DingDingMessageUtil.sendTextMessage(access_token, s) +//def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" +//DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("¸£ÖÝ", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index 7b357ab95..2cceebe1e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -143,9 +143,14 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){ retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price } + if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){ + retMap.anjuke_esf_price = d.get("anjuke_esf_price") + } + if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){ retMap.anjuke_esf_price = d.get("58tc_esf_price") } + if(!retMap.soufang_esf_price && d.soufang_esf_price){ retMap.soufang_esf_price = d.soufang_esf_price } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index e4b621c99..5da258e86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -165,7 +165,7 @@ class OpenApiUtils { return null; } address = address.trim().replace(" ","") - String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" + String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" List poiList = null; com.alibaba.fastjson.JSONObject result = null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy index 45c2f60a5..42dd5ea08 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy @@ -12,7 +12,8 @@ class OpenUserBatch { def task = new OpenUserBatch(); // task.run(); // task.test(); - task.addShenZhenFjtUser(); +// task.addShenZhenFjtUser(); + task.addFujianFjtUser() } def test(){ @@ -63,4 +64,23 @@ class OpenUserBatch { } } + def addFujianFjtUser(){ + def datas = ExcelUtils.read("D:\\Êý¾Ý\\½­ËÕÅ©ÐÐ\\¸£½¨Ê¡Å©ÐÐÕ˺ſªÍ¨.xlsx",0,1) + def i = 0; + for(def data : datas){ + i++; + def id = 252738 + i; + String mobile = data['A'] + String pwd = '111111' + def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '"+pwd+"', '12', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '100', NULL, NULL, '0', NULL, NULL, '100', '2022-07-10 00:00:00', '0', '', NULL)"; + onLineSqlInstance.execute(sql); + + def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data['C']+"', '¸£½¨', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '½¯Ïþ´º')"; + onLineSqlInstance.execute(sql2) + + def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'¸£ÖÝ,ÏÃÃÅ,ȪÖÝ,ÆÎÌï,ÄþµÂ,ÈýÃ÷,ÁúÑÒ,ÄÏÆ½,ÕÄÖÝ','¸£ÖÝ:,ÏÃÃÅ:,ȪÖÝ:,ÆÎÌï:,ÄþµÂ:,ÈýÃ÷:,ÁúÑÒ:,ÄÏÆ½:,ÕÄÖÝ:')" + onLineSqlInstance.execute(sql3) + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/AnjukeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/AnjukeTest.groovy new file mode 100644 index 000000000..e3ef4608c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/AnjukeTest.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.gj.abuyun + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class AnjukeTest { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeTest() + task.run() + } + + def run(){ + def q = new BasicDBObject() + def dbcursor = detail.find(q) + for(def d : dbcursor){ + def url = d.jumpAction + def s =ClaABuYun.getResponse(url) + println(s) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/ProxyAuthenticator.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/ProxyAuthenticator.groovy new file mode 100644 index 000000000..36115ae9c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abuyun/ProxyAuthenticator.groovy @@ -0,0 +1,23 @@ +package com.fangjia.script.gj.abuyun + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.net.Authenticator; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.net.URL; + +class ProxyAuthenticator extends Authenticator { + private String user, password; + + public ProxyAuthenticator(String user, String password) { + this.user = user; + this.password = password; + } + + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(user, password.toCharArray()); + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAbuyunCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAbuyunCrawl.groovy new file mode 100644 index 000000000..97dc61875 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAbuyunCrawl.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.abuyun.ClaABuYun +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup + + +class AnjukeAbuyunCrawl { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeAbuyunCrawl(); + task.run(); + } + def run(){ + def url = "https://fz.anjuke.com/sale/?from=navigation" + for(def i=2;i<50;i++){ + url ="https://fz.anjuke.com/sale/p${i}/?from=navigation" + def doc = getABuYunDocument(url); + def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) + println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles!=null?eles.size():0 + if(eles){ + parser(eles) + } + } + } + + def parser(eles){ + def index = 0 + for(def e : eles){ + index ++ + def href = e.attr("href") + def doc = getABuYunDocument(href); + def newUrl = href.substring(0, href.indexOf("?")) + sendMq(newUrl, doc) + println "[ "+index +"] "+ newUrl + } + } + + + + def getABuYunDocument(url){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + def s = ClaABuYun.getResponse(url) + doc = Jsoup.parse(s); + def els = JQueryUtils.select(".list-main .list-left .list", doc) + if(!els){ + els = JQueryUtils.select(".props-main", doc) + } + if(els && els.size()>0){ + stop = true; + }else{ + error++ + println "[${error}] error: "+url + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + + def sendMq(href, doc){ + JSON json = new JSONObject() + json.put("project", "sell_anjuke") + json.put("url", href) + + JSON result = new JSONObject() + result.put("content", doc.toString()) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index d55350598..5354eafe8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -1,31 +1,239 @@ package com.fangjia.script.gj.crawl +import com.alibaba.fastjson.JSON +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.Utils import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import net.sf.json.JSONObject +import org.apache.commons.lang.StringEscapeUtils import org.jsoup.Jsoup import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver +import javax.script.Invocable +import javax.script.ScriptEngine +import javax.script.ScriptEngineManager +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + class AnjukeAppCrawl { + static def PREFIX_UNICODE = "\\u"; public static final SHORT_DATE = Utils.getLastMonth(); def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); public static def SOURCE = "°²¾Ó¿Í"; DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + static main(args) { + + + +// String line2 = StringEscapeUtils.unescapeJava(s); +// println line2 def task = new AnjukeAppCrawl(); +// task.autoRun() // task.run(); // task.run2() task.addThirdPrice() +// task.saveToEs() +// task.saveToEs1() + +// task.groupByCity() +// task.selectNoSave() +// task.statCount() + } + def saveToEs1(){ + def lists = getAllAppDatas() + anjukeToDB(lists) + } + + def statCount(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\anjuke\\cur_city_count.xlsx",{d-> + def city = d['B'] + if(!d['D']){ + def q = new BasicDBObject() + q.put("base.cityName",city) + def count = detail.count(q) + d['D'] = count + println city +" "+count + } + + }) + } + + def selectNoSave(){ + def esData = findAllEsData() + def q = new BasicDBObject() + def dbcursor = detail.find(q).batchSize(5000) + def index = 0 + for(def d : dbcursor){ + index++ + if(!esData.contains(d.jumpAction)){ + println "û±£´æ "+ d.jumpAction + } + if(index%100000==0){ + println index + } + + } + } + + def findAllEsData(){ + def urls = new HashSet() + ElasticUtils.executeSqlQuery("select url from community where source='°²¾Ó¿ÍÒÆ¶¯¶Ë'", { + urls << it.url + }) + return urls; + } + + def groupByCity(){ + def q = new BasicDBObject() + def dbcursor = detail.find(q).batchSize(5000) + def map = [:] + def index = 0 + for(def d : dbcursor){ + index++ + def city = d.base.cityName + def str = map.get(city) + if(!str){ + map.put(city, d.base.longCityListName) + } + println index + } + + map.each {k,v-> + println k + } + } + + def autoRun(){ + def rets = ExcelUtils.read("d:\\Êý¾Ý\\°²¾Ó¿Í³ÇÊÐÇøÓò±àÂë.xlsx") + def total= rets.size() + def index =0 + for(def d : rets){ + index++ + println index+" "+ d['A']+" "+d['C'] + getDetailByRegion(d['A'], d['B'], d['D'], null) + } + + } + + def saveToEs(){ + def rets = ExcelUtils.read("d:\\Êý¾Ý\\°²¾Ó¿Í³ÇÊÐÇøÓò±àÂë.xlsx",2,0) + def citys = ["¸£ÖÝ"] + for(def d : rets){ + if(citys.contains(d['A'])){ + def lists = getAllAppDatas(d['A']) + anjukeToDB(lists) + citys << d['A'] + } + } + } + + def anjukeToDB(def lists){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(lists.size()) + for(def i=0;i + index++ + def regionDatas = getRegions(m.url+"community/?from=anjuke_home", driver) + println "[${index}] "+ch_city+" "+regionDatas.size() + regionDatas.each{k,v-> + + rets << [ch_city, m.city_id.toInteger(),v.info.name, v.info.value_str.toInteger()] + } + } + ExcelUtils.write("d:\\Êý¾Ý\\°²¾Ó¿Í³ÇÊÐÇøÓò±àÂë.xlsx", rets, "Êý¾Ý") } def run2(){ @@ -46,6 +254,58 @@ class AnjukeAppCrawl { } } + def getRegions(url,driver){ + def doc = getDocument(url, driver) + def start = doc.toString().indexOf("") + if(start>0 && end>0 && end > start){ + def str = doc.toString().substring(start, end).replace("