字段覆盖率修改

This commit is contained in:
2023-05-17 10:53:07 +08:00
parent e441a10add
commit cd4029bbb2
6 changed files with 85 additions and 38 deletions
@@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -1,5 +1,7 @@
package com.fangjia.script.dataAna package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import java.text.DecimalFormat import java.text.DecimalFormat
@@ -57,13 +59,15 @@ class CityInfoAnaBatch extends BaseBatch {
final static def CITYS = []; final static def CITYS = [];
//floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence',
public static final def DISTRICT_FIELDS_MAP = [ public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate', name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount' alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
] ]
public static final def OFFICE_FIELDS_MAP = [ public static final def OFFICE_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
@@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
// 当期周末 // 当期周末
Date weekend = DateUtil.getFirstDayOfWeek(new Date()); Date weekend = DateUtil.getFirstDayOfWeek(new Date());
//周日开始跑,需要跑加上7天,周一跑不需要 //周日开始跑,需要跑加上7天,周一跑不需要
@@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v -> allCitys.each {k,v ->
// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){ // if(["南京"].contains(k.toString())){
CITYS << k.toString(); CITYS << k.toString();
// } // }
} }
@@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4)) entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
} }
entity.put('district_detail', districtDetail(disCount, cityName)); entity.put('district_detail', districtDetail(disCount, cityName));
entity.put('district_alias_total_count', districtAliasStat(cityName));
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName)); entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
} else { } else {
entity.put('district_detail', [:]); entity.put('district_detail', [:]);
@@ -1032,11 +1037,30 @@ class CityInfoAnaBatch extends BaseBatch {
return CITY_INFO_CONN.findOne(query); return CITY_INFO_CONN.findOne(query);
} }
def districtDetail(def disCount, def cityName) { def districtAliasStat(def cityName){
DBObject query = new BasicDBObject(); DBObject query = new BasicDBObject();
query.put('city', cityName); query.put('city', cityName);
query.put('cat', 'district'); query.put('cat', 'district');
query.put('visible', 0); query.put('visible', 0);
DBObject fields = new BasicDBObject()
fields.put("alias", 1)
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def aliasCount = 0
while(iter.hasNext()) {
def dbo = iter.next()
if(dbo && dbo.alias){
aliasCount = aliasCount + dbo.alias.size()
}
}
return aliasCount;
}
def districtDetail(def disCount, def cityName) {
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'district');
query.put('status', 0);
DBObject fields = new BasicDBObject() DBObject fields = new BasicDBObject()
DISTRICT_FIELDS_MAP.each() {key,skey-> DISTRICT_FIELDS_MAP.each() {key,skey->
@@ -1046,7 +1070,6 @@ class CityInfoAnaBatch extends BaseBatch {
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
def countMap = [:] def countMap = [:]
while(iter.hasNext()) { while(iter.hasNext()) {
def dbo = iter.next() def dbo = iter.next()
@@ -78,4 +78,16 @@ class FileUtils {
} }
out.close(); out.close();
} }
static def readTxt(def file){
def f = new File(file)
def ret = [];
if(f.exists()){
def lines = org.apache.commons.io.FileUtils.readLines(f)
for(line in lines) {
ret << line;
}
}
return ret;
}
} }
@@ -44,7 +44,7 @@ for(def c : citys){
def v1 = weekendDetailMap.get(key) def v1 = weekendDetailMap.get(key)
def v2 = lastWeekendDetailMap.get(key) def v2 = lastWeekendDetailMap.get(key)
if(v1 && v2){ if(v1 && v2){
ret << DataHandleUtils.getDoubleRound(2, v1-v2) ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100
}else{ }else{
ret << "-" ret << "-"
} }
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
// //
// } // }
// citys = citys[(start+1)..(citys.size()-1)] // citys = citys[(start+1)..(citys.size()-1)]
driver = getLogin() driver = getLogin(driver)
crawlData(driver, citys) crawlData(driver, citys)
@@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome {
def s = driver.getPageSource() def s = driver.getPageSource()
def flag = true def flag = true
while(flag){ while(flag){
if(s && s.contains("无法访问此网站")){ if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin() driver = getLogin(driver)
driver.get(url) driver.get(url)
Thread.sleep(5000) Thread.sleep(5000)
s = driver.getPageSource() s = driver.getPageSource()
@@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome {
def detailUrls = [] def detailUrls = []
while(!stop){ while(!stop){
def url = cityUrl + "pg${page}/" def url = cityUrl + "pg${page}/"
try{ driver.get(url)
driver.get(url) // def doc = Jsoup.parse(driver.getPageSource());
}catch(Exception e){ def doc = checkDocument(driver, url);
println url +" 出错"
}
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc) def elements = JQueryUtils.select(".listContent li", doc)
if(elements){ if(elements){
for(def e : elements){ for(def e : elements){
@@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource()); def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = [] def list = []
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"] def citys = ["福州"]
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = "https:"+e.attr("href") def href = "https:"+e.attr("href")
@@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome {
return list return list
} }
def getLogin(){ def getLogin(driver){
WebDriver driver = new ChromeDriver() // WebDriver driver = new ChromeDriver()
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20) WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000) Thread.sleep(10000)
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
@@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome {
WebDriver driver = new ChromeDriver() WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver) def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse() // cityMap = cityMap.reverse()
driver = getLogin() driver = getLogin(driver, "https://sh.lianjia.com/")
crawlData(driver, cityMap) crawlData(driver, cityMap)
} }
def standardRegion(){ def standardRegion(){
@@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome {
def crawlData(driver, cityList){ def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
for(def ret : cityList){ for(def ret : cityList){
def city = ret[0] def city = ret[0]
if(runedCitys.contains(city)){
println city +" 已运行"
continue
}
def cityUrl = ret[1] def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/" def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl) driver.get(newUrl)
@@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome {
} }
} }
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true);
} }
} }
@@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome {
def s = driver.getPageSource() def s = driver.getPageSource()
def flag = true def flag = true
while(flag){ while(flag){
if(s && s.contains("无法访问此网站")){ if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin() driver = getLogin(driver, url)
driver.get(url) driver.get(url)
Thread.sleep(5000) Thread.sleep(5000)
s = driver.getPageSource() s = driver.getPageSource()
@@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome {
while(!stop){ while(!stop){
def url = cityUrl + "pg${page}/" def url = cityUrl + "pg${page}/"
driver.get(url) driver.get(url)
def doc = Jsoup.parse(driver.getPageSource()); //def detailUrls = getPageUrls(driver, cityUrl)
def doc = checkDocument(driver, url);
def elements = JQueryUtils.select(".listContent li", doc) def elements = JQueryUtils.select(".listContent li", doc)
if(elements){ if(elements){
for(def e : elements){ for(def e : elements){
@@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource()); def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityList = [] def cityList = []
def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"] def citys= ["福州"]
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
@@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome {
return cityList return cityList
} }
def getLogin(){ def getLogin(driver, url){
WebDriver driver = new ChromeDriver() // WebDriver driver = new ChromeDriver()
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20) WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000) Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); try{
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
}catch(Exception e){
Thread.sleep(60000)
getLogin(driver, url)
}
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框