From cd4029bbb25fdade71352882f0eee7d9ea477089 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 17 May 2023 10:53:07 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AD=97=E6=AE=B5=E8=A6=86=E7=9B=96=E7=8E=87?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/AddSzDealData.groovy | 3 ++ .../script/dataAna/CityInfoAnaBatch.groovy | 47 ++++++++++++++----- .../fangjia/script/gj/ccb/FileUtils.groovy | 12 +++++ .../gj/district/DistrictFiledStat.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 22 ++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 37 ++++++++++----- 6 files changed, 85 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 0a601a6ec..962d75c3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + * 深圳公众号成交数据录入 + */ def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 45da16ec1..7447a6712 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.dataAna +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.MongoUtils import java.text.DecimalFormat @@ -56,14 +58,16 @@ class CityInfoAnaBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) final static def CITYS = []; - + + //floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence', public static final def DISTRICT_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', - start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', - alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount' + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', + start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', + township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', @@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch { @Override public void process(Object args) { + // 当期周末 Date weekend = DateUtil.getFirstDayOfWeek(new Date()); //周日开始跑,需要跑加上7天,周一跑不需要 @@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> -// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){ +// if(["南京"].contains(k.toString())){ CITYS << k.toString(); // } } @@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch { entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4)) } entity.put('district_detail', districtDetail(disCount, cityName)); - + entity.put('district_alias_total_count', districtAliasStat(cityName)); entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName)); } else { entity.put('district_detail', [:]); @@ -1031,22 +1036,40 @@ class CityInfoAnaBatch extends BaseBatch { query.put("weekend", lastWeekend); return CITY_INFO_CONN.findOne(query); } + + def districtAliasStat(def cityName){ + DBObject query = new BasicDBObject(); + query.put('city', cityName); + query.put('cat', 'district'); + query.put('visible', 0); + DBObject fields = new BasicDBObject() + fields.put("alias", 1) + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + Iterator iter = cur.iterator() + def aliasCount = 0 + while(iter.hasNext()) { + def dbo = iter.next() + if(dbo && dbo.alias){ + aliasCount = aliasCount + dbo.alias.size() + } + } + return aliasCount; + } def districtDetail(def disCount, def cityName) { DBObject query = new BasicDBObject(); query.put('city', cityName); query.put('cat', 'district'); - query.put('visible', 0); + query.put('status', 0); DBObject fields = new BasicDBObject() DISTRICT_FIELDS_MAP.each() {key,skey-> fields.put(key, 1) } - + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] - while(iter.hasNext()) { def dbo = iter.next() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 23513b80b..30a4e51ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -78,4 +78,16 @@ class FileUtils { } out.close(); } + + static def readTxt(def file){ + def f = new File(file) + def ret = []; + if(f.exists()){ + def lines = org.apache.commons.io.FileUtils.readLines(f) + for(line in lines) { + ret << line; + } + } + return ret; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy index 47f1563d0..e3d7eb51c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy @@ -44,7 +44,7 @@ for(def c : citys){ def v1 = weekendDetailMap.get(key) def v2 = lastWeekendDetailMap.get(key) if(v1 && v2){ - ret << DataHandleUtils.getDoubleRound(2, v1-v2) + ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100 }else{ ret << "-" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ca92bb2c3..f71b47a56 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome { // // } // citys = citys[(start+1)..(citys.size()-1)] - driver = getLogin() + driver = getLogin(driver) crawlData(driver, citys) @@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome { def s = driver.getPageSource() def flag = true while(flag){ - if(s && s.contains("无法访问此网站")){ - driver = getLogin() + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - try{ - driver.get(url) - }catch(Exception e){ - println url +" 出错" - } - - def doc = Jsoup.parse(driver.getPageSource()); + driver.get(url) +// def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, url); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) def list = [] - def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"] + def citys = ["福州"] if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") @@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(){ - WebDriver driver = new ChromeDriver() + def getLogin(driver){ +// WebDriver driver = new ChromeDriver() driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index bc5448c6c..a2e8be163 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome { WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() - driver = getLogin() + driver = getLogin(driver, "https://sh.lianjia.com/") crawlData(driver, cityMap) - - } def standardRegion(){ @@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome { def crawlData(driver, cityList){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt") + for(def ret : cityList){ def city = ret[0] + if(runedCitys.contains(city)){ + println city +" 已运行" + continue + } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" driver.get(newUrl) @@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome { } } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + + FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true); } } @@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome { def s = driver.getPageSource() def flag = true while(flag){ - if(s && s.contains("无法访问此网站")){ - driver = getLogin() + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome { while(!stop){ def url = cityUrl + "pg${page}/" driver.get(url) - def doc = Jsoup.parse(driver.getPageSource()); + //def detailUrls = getPageUrls(driver, cityUrl) + def doc = checkDocument(driver, url); def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityList = [] - def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") @@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome { return cityList } - def getLogin(){ - WebDriver driver = new ChromeDriver() - driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); + def getLogin(driver, url){ +// WebDriver driver = new ChromeDriver() + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) + driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) - wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); - + try{ + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + }catch(Exception e){ + Thread.sleep(60000) + getLogin(driver, url) + } WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框