From 240eb5f22a55a9159337487c57606d50a9c13604 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 11 May 2023 10:08:40 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E6=B7=BB=E5=8A=A02=E4=B8=AA=E6=9D=A5?= =?UTF-8?q?=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/mongo/fangjia/enums/NewResourceRecommendEnum.java | 4 +++- .../com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java index 8aea43b8e..e53f9ba4e 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/NewResourceRecommendEnum.java @@ -42,7 +42,9 @@ public enum NewResourceRecommendEnum { GOV("1034", "政府机构"), BOND("1033", "证券"), COST("1035", "消费金融"), - ZIXUN_APP("1036", "业务咨询app"); + ZIXUN_APP("1036", "业务咨询app"), + ARTICLE_APP("1037", "公众号推文"), + MENU_APP("1038", "公众号菜单"); NewResourceRecommendEnum(String id, String value) { this.id = id; diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java index db571adae..3310d6a09 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java @@ -38,7 +38,9 @@ public enum SalesLeadEnum { GOV("1034", "政府机构"), BOND("1033", "证券"), COST("1035", "消费金融"), - ZIXUN_APP("1036", "业务咨询app"); + ZIXUN_APP("1036", "业务咨询app"), + ARTICLE_APP("1037", "公众号推文"), + MENU_APP("1038", "公众号菜单"); SalesLeadEnum(String id, String value) { this.id = id; From eb80d9401f9994f6ae19203f791003ba3e81f845 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 10:23:50 +0800 Subject: [PATCH 2/5] update --- .../ronghe/ExportDistrictRongHeTest.groovy | 50 +-- .../com/fangjia/script/gj/EsHouseStat.groovy | 20 +- .../com/fangjia/script/gj/FiledStrand.groovy | 78 ++++- .../gj/district/DistrictFieldAddBatch.groovy | 320 ++++++++++++------ .../gj/district/DistrictFiledStat.groovy | 69 ++++ .../script/gj/es/EsMonthHouseStat.groovy | 39 ++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 12 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 6 +- .../script/trade/OpenUriUserIdCountDay.groovy | 10 +- 9 files changed, 454 insertions(+), 150 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 0f2519b79..66f2cc491 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{ // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// if(curDate.toString().endsWith("10")){ + if(curDate.toString().endsWith("10")){ -// exportEvaluateRate() -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// -// -// exportBigdataCommunityGJRH()/ -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// -// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()/ + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) exportInOk() -// } + } SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index b78e0ec4b..da0c07889 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -29,16 +29,16 @@ class EsHouseStat { DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) // DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020) -// public static JdbcDataSource dataSource = new JdbcDataSource() -// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" -// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) -// static { -// dataSource.setUrl(address) -// Properties connectionProperties = new Properties() -// connectionProperties.put("user", "developer") -// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") -// dataSource.setProperties(connectionProperties) -// } + public static JdbcDataSource dataSource = new JdbcDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 805c41944..e6cdef564 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -285,7 +285,7 @@ class FiledStrand { } static def processCarSiteDesc(car_site){ - if(car_site && car_site.size()<=20){ + if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){ return car_site.toString() } return null @@ -337,6 +337,82 @@ class FiledStrand { return null } + static def processBlock(def block){ + if(block){ + if(block.contains("无") || block.toString().length()>10){ + return null + } + return block + } + return null + } + + static def processBuildType(def buildType, def estateType){ + if(buildType){ + def types = [] + if(estateType == "别墅"){ + def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + }else if(estateType == "商住"){ + def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + }else{ + def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"] + for(def s : arrs){ + if(buildType.contains(s)){ + types << s + } + } + } + if(types){ + return types.join(",").toString() + } + } + return null + } + + static def processlandProperty(val){ + if(val){ + def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"] + def property = null + for(def s : arrs){ + if(val.contains(s)){ + property = s + break; + } + } + return property + } + return null + } + + static def processBuildStructure(val) { + if (val) { + def buildStructure= null + if(val.indexOf("砖木")>-1){ + buildStructure = "砖木结构" + }else if(val.indexOf("钢结构")>-1){ + buildStructure = "钢结构"; + }else if(val.indexOf("钢、钢筋混凝土")>-1){ + buildStructure = "钢、钢筋混凝土结构"; + }else if(val.indexOf("钢筋混凝土")>-1){ + buildStructure = "钢筋混凝土结构"; + }else if(val.indexOf("混合")>-1){ + buildStructure = "混合结构"; + } + return buildStructure + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index d32f3e459..5495b6bfd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.FiledStrand import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.Jms import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -17,6 +19,7 @@ import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId +import java.text.SimpleDateFormat import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -31,15 +34,32 @@ import java.util.regex.Pattern class DistrictFieldAddBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) - + static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018) //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" - //,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家" - def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源 + // + def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 //"alias":"alias", - def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段 + //ownership_type:权属类型 为商品房、经适房、动迁房等 + //house_property 产权性质:国有产权 "house_property":"网站没有", + def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", + "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", + "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", + "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段 + + def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商", + "virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型", + "car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构", + "build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"] + + + final static def EMAILS = [ + "gaojuan@fangjia.com" + ] static main(args) { runBatch(new DistrictFieldAddBatch(), args) @@ -47,21 +67,37 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京" - + def citys = CommonDatautils.findAllCity() def analysRets = [] + + def rh_analysRets = [] + def fields = fieldsMap.keySet() as List + def headers = ["来源", "城市"] + fields.each{it-> + headers << fieldsNameMap.get(it) + } + + headers << "抓取量" + headers << "匹配量" + headers << "0.9分以上量" + headers << "匹配率" + headers << "0.9以上匹配率" +// analysRets << headers + for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) -// addFieldFromRonghe(c, totalCount) + rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets) } println analysRets.toString() - - ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") + sendStatEmail(headers, analysRets, rh_analysRets) +// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") } def addFieldFromMatch(c, totalCount, analysRets){ + def fields = fieldsMap.keySet() as List + sourceMap.each { seaweedmapSource, esWebSource -> def esRets = findEsDatas(c, esWebSource) def matchRets1 = matchApi(esRets) @@ -80,7 +116,18 @@ class DistrictFieldAddBatch extends BaseBatch{ pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size()) pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size()) } - analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2] + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analyRet << matchRets1.size() + analyRet << matchRets2.size() + analyRet << pre1 + analyRet << pre2 + analysRets << analyRet } return analysRets; } @@ -88,7 +135,8 @@ class DistrictFieldAddBatch extends BaseBatch{ - def addFieldFromRonghe(c, totalCount){ + def addFieldFromRonghe(c, totalCount, rh_analysRets){ + def fields = fieldsMap.keySet() as List sourceMap.each{seaweedmapSource,esWebSource-> def rhRets = findRongheDists(c, seaweedmapSource) def esRets = findEsDistrict(c, esWebSource) @@ -99,7 +147,26 @@ class DistrictFieldAddBatch extends BaseBatch{ println c +" "+precentMap.toString() updateDB(c, updateRets) println "----------------------------------------------------------------------------------" + def pre1 = 0 + def pre2 = 0 + if(esRets){ + pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size()) + pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size()) + } + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analyRet << rhRets.size() + analyRet << esRets.size() + analyRet << pre1 + analyRet << pre2 + rh_analysRets << analyRet } + return rh_analysRets; } def processFieldsFromRh(c, rhRets, esRets){ @@ -116,26 +183,7 @@ class DistrictFieldAddBatch extends BaseBatch{ println c+" "+rhDist.id+" 不存在[ES]" continue } - def newValues = [:] - fieldsMap.each{seaweedfield,esField-> - def val = dist.get(seaweedfield) - if(!val){ - def esVal = esDbo.get(esField) - if(esVal){ - def newVal = null - if(seaweedfield == "estate_charge"){ - newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) - newVal = newVal.toString() - } - if(seaweedfield == "build_date"){ - newVal = FiledStrand.processFinishBuildingDate(esVal) - } - if(newVal){ - newValues.put(seaweedfield, newVal) - } - } - } - } + def newValues = strandField(c, dist, esDbo) if(newValues){ fieldValueMap.put(dist._id.toString(), newValues) } @@ -149,77 +197,15 @@ class DistrictFieldAddBatch extends BaseBatch{ def processFieldsFromMatch(c,matchRets){ def fields = fieldsMap.keySet() as List fields << "city" + fields << "estate_type2" def fieldValueMap = [:] for(def matchDist : matchRets){ - def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ println c+" "+matchDist.match_id+" 不存在" continue } - - def newValues = [:] - fieldsMap.each{seaweedfield,esField-> - def val = dist.get(seaweedfield) - if(!val){ - def esVal = matchDist.get(esField) - if(esVal){ - def newVal = null - if(seaweedfield == "estate_charge"){ - newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) - newVal = newVal.toString() - } - if(seaweedfield == "build_date"){ - newVal = FiledStrand.processFinishBuildingDate(esVal) - } - if(seaweedfield == "estate_company" || seaweedfield == "developer"){ - newVal = FiledStrand.processCompany(esVal) - } - if(seaweedfield =="virescence_percent"){ - newVal = FiledStrand.processVirescencePercent(esVal) - } - - if(seaweedfield =="cubage_percent"){ - newVal = FiledStrand.processCubagePercent(esVal) - } - - if(seaweedfield =="all_area" || seaweedfield =="build_area"){ - newVal = FiledStrand.processArea(esVal) - newVal = newVal.toString() - } - if(seaweedfield =="household_count"){ - newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) - } - - if(seaweedfield =="loudong_count"){ - newVal = FiledStrand.processLoudongCount(esVal) - } - - if(seaweedfield =="car_site_count"){ - newVal = FiledStrand.processCarSite(esVal) - } - if(seaweedfield =="car_site"){ - newVal = FiledStrand.processCarSiteDesc(esVal) - } - - if(seaweedfield =="property_life"){ - newVal = FiledStrand.processPropertyLife(esVal) - } - - if(seaweedfield =="alias"){ - newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) - } - - if(seaweedfield =="address"){ - newVal = FiledStrand.processAddress(esVal, dist.city) - } - - if(newVal){ - newValues.put(seaweedfield, newVal) - } - } - } - } + def newValues = strandField(c, dist, matchDist) if(newValues){ println dist._id.toString()+" "+newValues.toString() fieldValueMap.put(dist._id.toString(), newValues) @@ -229,6 +215,89 @@ class DistrictFieldAddBatch extends BaseBatch{ return fieldValueMap; } + def strandField(c,dist, matchDist){ + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = dist.get(seaweedfield) + if(!val){ + def esVal = matchDist.get(esField) + if(esVal){ + def newVal = null + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + newVal = newVal.toString() + } + if(seaweedfield == "build_date" || seaweedfield == "live_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "estate_company" || seaweedfield == "developer"){ + newVal = FiledStrand.processCompany(esVal) + } + if(seaweedfield =="virescence_percent"){ + newVal = FiledStrand.processVirescencePercent(esVal) + } + + if(seaweedfield =="cubage_percent"){ + newVal = FiledStrand.processCubagePercent(esVal) + } + + if(seaweedfield =="all_area" || seaweedfield =="build_area"){ + newVal = FiledStrand.processArea(esVal) + if(newVal){ + newVal = newVal.toString() + } + } + if(seaweedfield =="household_count"){ + newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) + } + + if(seaweedfield =="loudong_count"){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(seaweedfield =="car_site_count"){ + newVal = FiledStrand.processCarSite(esVal) + } + if(seaweedfield =="car_site"){ + newVal = FiledStrand.processCarSiteDesc(esVal) + } + + if(seaweedfield =="property_life"){ + newVal = FiledStrand.processPropertyLife(esVal) + } + + if(seaweedfield =="alias"){ + newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) + } + + if(seaweedfield =="address"){ + newVal = FiledStrand.processAddress(esVal, dist.city) + } + if(seaweedfield =="block"){ + newVal = FiledStrand.processBlock(esVal) + } + + if(seaweedfield == "build_type"){ + newVal = FiledStrand.processBuildType(esVal, dist.estate_type2) + } + + if(seaweedfield == "land_property"){ + newVal = FiledStrand.processlandProperty(esVal) + } + + if(seaweedfield == "build_structure"){ + newVal = FiledStrand.processBuildStructure(esVal) + } + + if(newVal){ + newValues.put(seaweedfield, newVal) + } + } + } + } + return newValues; + } + def findRongheDists(def city, def source){ @@ -285,9 +354,9 @@ class DistrictFieldAddBatch extends BaseBatch{ def precentMap = [:] fieldsMap.each() {key,skey-> if(countMap.containsKey(key)) { - precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) + precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) }else { - precentMap.put(skey, 0) + precentMap.put(key, 0) } } return precentMap; @@ -414,4 +483,61 @@ class DistrictFieldAddBatch extends BaseBatch{ } return false } + + def sendStatEmail(headers, analysRets, rh_nalysRets){ + Date dateTime = DateUtils.mins(1); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd"); + StringBuffer sb = new StringBuffer(); + if(analysRets){ + sb.append("") + sb.append("字段补充情况分析") + sb.append("") + headers.each{it-> + if(it=="来源" || it=="城市"){ + sb.append("") + }else{ + sb.append("") + } + } + sb.append("") + + for(def ret : analysRets){ + sb.append(""); + ret.each{it-> + sb.append("") + } + sb.append(""); + } + sb.append("
"+ it+""+ it+"
" + it + "

"); + } + + + StringBuffer rh_sb = new StringBuffer(); + if(rh_nalysRets){ + rh_sb.append("") + rh_sb.append("字段补充情况分析") + rh_sb.append("") + headers.each{it-> + if(it=="来源" || it=="城市"){ + rh_sb.append("") + }else{ + rh_sb.append("") + } + } + rh_sb.append("") + + for(def ret : rh_nalysRets){ + rh_sb.append(""); + ret.each{it-> + rh_sb.append("") + } + rh_sb.append(""); + } + rh_sb.append("
"+ it+""+ it+"
" + it + "

"); + } + + for (def email : EMAILS) { + jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString()); + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy new file mode 100644 index 000000000..47f1563d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy @@ -0,0 +1,69 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 字段提升率统计 + */ +DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + +def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ] +def weeknd = DateUtils.getFirstDayOfWeek(new Date()) +def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7) + + +def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd)) +def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd)) + +def weekendMapDatas = findDataByCity(weekendDatas) +def lastWeekendMapDatas = findDataByCity(lastWeekendDatas) + +def citys = CommonDatautils.findAllCity() + +def fields = fieldsMap.keySet() as List +def rets = [] +def headers = [] +headers << "城市" +fields.each{k-> + headers << fieldsMap.get(k) +} +rets << headers +for(def c : citys){ + def weekendDetailMap = weekendMapDatas.get(c) + def lastWeekendDetailMap = lastWeekendMapDatas.get(c) + def ret = [] + ret << c + fields.each{key-> + if(weekendDetailMap){ + def v1 = weekendDetailMap.get(key) + def v2 = lastWeekendDetailMap.get(key) + if(v1 && v2){ + ret << DataHandleUtils.getDoubleRound(2, v1-v2) + }else{ + ret << "-" + } + }else{ + ret << "-" + } + } + rets << ret +} + +ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计") + + + +def findDataByCity(weekendDatas){ + def map = [:] + for(def d : weekendDatas){ + d.district_detail.district_count = d.existing_district_count + map.put(d.city, d.district_detail) + } + return map +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index a3e22d7a6..9be75aeb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.gj.es +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -19,11 +21,44 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() - task.houseRun() +// task.houseRun() // task.statHouseUpdateCount() // task.statDistHouse() + task.statRent() + } + + def statRent(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d -> + index++ + def region = RegionData.formatRegion(d['A'], d['B']) + if(region){ + def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(!dist){ + dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region) + if(dist){ + dist = CommonDatautils.findDistrictById(dist.id) + } + } + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501) + d['G'] = distCount + d['E'] = dist.region + d['F'] = dist.name + println index+" "+ d['C'] +" "+distCount + } + }); } def statDistHouse(){ @@ -172,7 +207,7 @@ class EsMonthHouseStat { def total = 0 try{ aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'" ElasticUtils.executeSqlQuery(sql,{data-> total = data.c diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 2aae0edb5..916db9961 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -239,6 +239,12 @@ public class BeiKeCrawlByChrome { if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } + if(!detail.area){ + areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "") + if (areaStr && areaStr.isNumber()) { + detail.area = areaStr.toDouble() + } + } def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { @@ -254,7 +260,7 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - if(detail.total_price){ + if(detail.total_price && detail.avg_price){ def detailUrl = e.select(".title a").attr("href") detail.url = detailUrl def dbo = findTrade(detail) @@ -404,9 +410,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") - if (citys.contains(e.text())){ +// if (citys.contains(e.text())){ list << [e.text(), href] - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 343a4a3a4..6c01e6c11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -333,7 +333,7 @@ public class LianjiaCrawlByChrome { detail.source = "链家在线" detail.url = e.select(".title a").attr("href") - if(detail.total_price){ + if(detail.total_price && detail.avg_price){ def dbo = findTrade(detail) if (!dbo) { def houseInfo = e.select(".houseInfo").text() @@ -482,9 +482,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ +// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] - } +// } } } return cityList diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy index 4e27d62a5..954fface3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy @@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch { runBatch(new OpenUriUserIdCountDay(), args) } final static def EMAILS = [ - "joe@fangjia.com", "wangwenyan@fangjia.com", "xiaohai@fangjia.com", "caoyuyan@fangjia.com", "jiangxiaochun@fangjia.com", - "gaojuan@fangjia.com", - 'shijing@fangjia.com', - 'zhongjingyi@fangjia.com', - "duweiliang@fangjia.com", - "sunjian@fangjia.com", - "yaojingmeng@fangjia.com", - "xuqi@fangjia.com", - "wangpeili@fangjia.com" + "gaojuan@fangjia.com" ] @Override From 0e26de8cad44e2273c099232f52019fb41e30c4e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 10:25:43 +0800 Subject: [PATCH 3/5] update --- .../com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 5495b6bfd..7c89352be 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def analysRets = [] From b9780aae68ee939af5cd3179878b8d8b7c12d471 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 11:10:01 +0800 Subject: [PATCH 4/5] update --- .../fangjia/script/gj/district/DistrictFieldAddBatch.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 7c89352be..f3a47806d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -87,8 +87,10 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) - rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) - analysRets = addFieldFromMatch(c, totalCount, analysRets) + if(totalCount){ + rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) + analysRets = addFieldFromMatch(c, totalCount, analysRets) + } } println analysRets.toString() sendStatEmail(headers, analysRets, rh_analysRets) From e441a10add801c62b94ab3255ab9622ccb82dd6c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 18:36:11 +0800 Subject: [PATCH 5/5] update --- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../gj/district/DistrictFieldAddBatch.groovy | 5 ++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 44 ++++++++++++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 31 ++++++++++--- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index e26dc05dd..a3be166aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index f3a47806d..2806a48f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -88,7 +88,7 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ - rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) +// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets) } } @@ -175,6 +175,9 @@ class DistrictFieldAddBatch extends BaseBatch{ def fields = fieldsMap.keySet() as List def fieldValueMap = [:] for(def rhDist : rhRets){ + if(rhDist.commid && rhDist.commid!="null"){ + continue + } def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields) if(!dist){ println c+" "+rhDist.commid+" 不存在" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 916db9961..ca92bb2c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome { // // } // citys = citys[(start+1)..(citys.size()-1)] - getLogin(driver) + driver = getLogin() crawlData(driver, citys) @@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome { } } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } + def crawlData(driver, citys){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ def city = m[0] def url = m[1] - driver.get(url+"/chengjiao/") + def newUrl = url+"/chengjiao/" + driver.get(newUrl) Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome { } def roomStr = arrs[1] + if(roomStr && !roomStr.contains("室")){ + roomStr = arrs[2] + } if (roomStr) { if (roomStr.indexOf("室") > 0) { roomStr = roomStr.substring(0, roomStr.indexOf("室")) @@ -235,16 +255,16 @@ public class BeiKeCrawlByChrome { } } - def areaStr = arrs[2].replace("平米", "") + def areaStr = arrs[2] + if(areaStr && !areaStr.contains("平米")){ + if(arrs.size()>=4){ + areaStr = arrs[3] + } + } + areaStr = areaStr.replace("平米","") if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } - if(!detail.area){ - areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "") - if (areaStr && areaStr.isNumber()) { - detail.area = areaStr.toDouble() - } - } def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { @@ -418,7 +438,8 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -438,6 +459,7 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) + return driver } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6c01e6c11..bc5448c6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome { WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() - getLogin(driver) + driver = getLogin() crawlData(driver, cityMap) @@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome { for(def ret : cityList){ def city = ret[0] def cityUrl = ret[1] + def newUrl = cityUrl+"chengjiao/" + driver.get(newUrl) //def detailUrls = getPageUrls(driver, cityUrl) - driver.get(cityUrl+"chengjiao/") Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome { return detailUrls } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } def getPageDeail(driver, city, cityUrl, regionText){ def stop = false @@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome { def parse(city,e, regionText){ try { - def title = e.select(".title a").text() + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") @@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome { } } - def price = e.select(".totalPrice .number").text() + def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { detail.total_price = (int)(price.toDouble() * 10000) } @@ -490,7 +507,8 @@ public class LianjiaCrawlByChrome { return cityList } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(20000) + return driver } }