字段覆盖率修改
This commit is contained in:
+3
@@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
+35
-12
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.dataAna
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
|
||||
import java.text.DecimalFormat
|
||||
@@ -56,14 +58,16 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
final static def CITYS = [];
|
||||
|
||||
|
||||
//floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence',
|
||||
public static final def DISTRICT_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount'
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||
]
|
||||
public static final def OFFICE_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
||||
@@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// 当期周末
|
||||
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
|
||||
//周日开始跑,需要跑加上7天,周一跑不需要
|
||||
@@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
||||
|
||||
allCitys.each {k,v ->
|
||||
// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){
|
||||
// if(["南京"].contains(k.toString())){
|
||||
CITYS << k.toString();
|
||||
// }
|
||||
}
|
||||
@@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
|
||||
}
|
||||
entity.put('district_detail', districtDetail(disCount, cityName));
|
||||
|
||||
entity.put('district_alias_total_count', districtAliasStat(cityName));
|
||||
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
|
||||
} else {
|
||||
entity.put('district_detail', [:]);
|
||||
@@ -1031,22 +1036,40 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
query.put("weekend", lastWeekend);
|
||||
return CITY_INFO_CONN.findOne(query);
|
||||
}
|
||||
|
||||
def districtAliasStat(def cityName){
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('cat', 'district');
|
||||
query.put('visible', 0);
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("alias", 1)
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def aliasCount = 0
|
||||
while(iter.hasNext()) {
|
||||
def dbo = iter.next()
|
||||
if(dbo && dbo.alias){
|
||||
aliasCount = aliasCount + dbo.alias.size()
|
||||
}
|
||||
}
|
||||
return aliasCount;
|
||||
}
|
||||
|
||||
def districtDetail(def disCount, def cityName) {
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put('city', cityName);
|
||||
query.put('cat', 'district');
|
||||
query.put('visible', 0);
|
||||
query.put('status', 0);
|
||||
|
||||
DBObject fields = new BasicDBObject()
|
||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||
fields.put(key, 1)
|
||||
}
|
||||
|
||||
|
||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
def countMap = [:]
|
||||
|
||||
while(iter.hasNext()) {
|
||||
def dbo = iter.next()
|
||||
|
||||
|
||||
@@ -78,4 +78,16 @@ class FileUtils {
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
static def readTxt(def file){
|
||||
def f = new File(file)
|
||||
def ret = [];
|
||||
if(f.exists()){
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(f)
|
||||
for(line in lines) {
|
||||
ret << line;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ for(def c : citys){
|
||||
def v1 = weekendDetailMap.get(key)
|
||||
def v2 = lastWeekendDetailMap.get(key)
|
||||
if(v1 && v2){
|
||||
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
|
||||
ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100
|
||||
}else{
|
||||
ret << "-"
|
||||
}
|
||||
|
||||
+9
-13
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
driver = getLogin()
|
||||
driver = getLogin(driver)
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
@@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome {
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
@@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
try{
|
||||
driver.get(url)
|
||||
}catch(Exception e){
|
||||
println url +" 出错"
|
||||
}
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
driver.get(url)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, url);
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
|
||||
def citys = ["福州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
@@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome {
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def getLogin(driver){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
|
||||
+25
-12
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome {
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
driver = getLogin()
|
||||
driver = getLogin(driver, "https://sh.lianjia.com/")
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def standardRegion(){
|
||||
@@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome {
|
||||
def crawlData(driver, cityList){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
|
||||
|
||||
for(def ret : cityList){
|
||||
def city = ret[0]
|
||||
if(runedCitys.contains(city)){
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
driver.get(newUrl)
|
||||
@@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome {
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
@@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome {
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def doc = checkDocument(driver, url);
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityList = []
|
||||
def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"]
|
||||
def citys= ["福州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
@@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome {
|
||||
return cityList
|
||||
}
|
||||
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
try{
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
}catch(Exception e){
|
||||
Thread.sleep(60000)
|
||||
getLogin(driver, url)
|
||||
}
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
|
||||
Reference in New Issue
Block a user