字段覆盖率修改

This commit is contained in:
2023-05-17 10:53:07 +08:00
parent e441a10add
commit cd4029bbb2
6 changed files with 85 additions and 38 deletions
@@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -1,5 +1,7 @@
package com.fangjia.script.dataAna
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import java.text.DecimalFormat
@@ -56,14 +58,16 @@ class CityInfoAnaBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final static def CITYS = [];
//floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence',
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount'
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
public static final def OFFICE_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
@@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
@Override
public void process(Object args) {
// 当期周末
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
//周日开始跑,需要跑加上7天,周一跑不需要
@@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v ->
// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){
// if(["南京"].contains(k.toString())){
CITYS << k.toString();
// }
}
@@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
}
entity.put('district_detail', districtDetail(disCount, cityName));
entity.put('district_alias_total_count', districtAliasStat(cityName));
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
} else {
entity.put('district_detail', [:]);
@@ -1031,22 +1036,40 @@ class CityInfoAnaBatch extends BaseBatch {
query.put("weekend", lastWeekend);
return CITY_INFO_CONN.findOne(query);
}
def districtAliasStat(def cityName){
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'district');
query.put('visible', 0);
DBObject fields = new BasicDBObject()
fields.put("alias", 1)
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def aliasCount = 0
while(iter.hasNext()) {
def dbo = iter.next()
if(dbo && dbo.alias){
aliasCount = aliasCount + dbo.alias.size()
}
}
return aliasCount;
}
def districtDetail(def disCount, def cityName) {
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'district');
query.put('visible', 0);
query.put('status', 0);
DBObject fields = new BasicDBObject()
DISTRICT_FIELDS_MAP.each() {key,skey->
fields.put(key, 1)
}
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
while(iter.hasNext()) {
def dbo = iter.next()
@@ -78,4 +78,16 @@ class FileUtils {
}
out.close();
}
static def readTxt(def file){
def f = new File(file)
def ret = [];
if(f.exists()){
def lines = org.apache.commons.io.FileUtils.readLines(f)
for(line in lines) {
ret << line;
}
}
return ret;
}
}
@@ -44,7 +44,7 @@ for(def c : citys){
def v1 = weekendDetailMap.get(key)
def v2 = lastWeekendDetailMap.get(key)
if(v1 && v2){
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100
}else{
ret << "-"
}
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
driver = getLogin()
driver = getLogin(driver)
crawlData(driver, citys)
@@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome {
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
@@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome {
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
try{
driver.get(url)
}catch(Exception e){
println url +" 出错"
}
def doc = Jsoup.parse(driver.getPageSource());
driver.get(url)
// def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, url);
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
def citys = ["福州"]
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
@@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome {
return list
}
def getLogin(){
WebDriver driver = new ChromeDriver()
def getLogin(driver){
// WebDriver driver = new ChromeDriver()
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
@@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome {
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
driver = getLogin()
driver = getLogin(driver, "https://sh.lianjia.com/")
crawlData(driver, cityMap)
}
def standardRegion(){
@@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome {
def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
for(def ret : cityList){
def city = ret[0]
if(runedCitys.contains(city)){
println city +" 已运行"
continue
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl)
@@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome {
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true);
}
}
@@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome {
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
@@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome {
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
//def detailUrls = getPageUrls(driver, cityUrl)
def doc = checkDocument(driver, url);
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityList = []
def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"]
def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome {
return cityList
}
def getLogin(){
WebDriver driver = new ChromeDriver()
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
def getLogin(driver, url){
// WebDriver driver = new ChromeDriver()
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
try{
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
}catch(Exception e){
Thread.sleep(60000)
getLogin(driver, url)
}
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框