字段覆盖率修改
This commit is contained in:
+3
@@ -6,6 +6,9 @@ import com.fangjia.script.util.MongoUtils
|
|||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 深圳公众号成交数据录入
|
||||||
|
*/
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|||||||
+33
-10
@@ -1,5 +1,7 @@
|
|||||||
package com.fangjia.script.dataAna
|
package com.fangjia.script.dataAna
|
||||||
|
|
||||||
|
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
|
||||||
import java.text.DecimalFormat
|
import java.text.DecimalFormat
|
||||||
@@ -57,13 +59,15 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
|
|
||||||
final static def CITYS = [];
|
final static def CITYS = [];
|
||||||
|
|
||||||
|
//floorplan_pics :'floorplanPics',out_pics :'outPics',out_pics : 'outPics', pics : 'pics',licence :'licence',
|
||||||
public static final def DISTRICT_FIELDS_MAP = [
|
public static final def DISTRICT_FIELDS_MAP = [
|
||||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
|
||||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||||
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
|
||||||
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
|
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
|
||||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
|
||||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount'
|
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||||
|
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||||
]
|
]
|
||||||
public static final def OFFICE_FIELDS_MAP = [
|
public static final def OFFICE_FIELDS_MAP = [
|
||||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
||||||
@@ -109,6 +113,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
|
||||||
// 当期周末
|
// 当期周末
|
||||||
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
|
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
|
||||||
//周日开始跑,需要跑加上7天,周一跑不需要
|
//周日开始跑,需要跑加上7天,周一跑不需要
|
||||||
@@ -123,7 +128,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
|
||||||
|
|
||||||
allCitys.each {k,v ->
|
allCitys.each {k,v ->
|
||||||
// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){
|
// if(["南京"].contains(k.toString())){
|
||||||
CITYS << k.toString();
|
CITYS << k.toString();
|
||||||
// }
|
// }
|
||||||
}
|
}
|
||||||
@@ -574,7 +579,7 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
|
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
|
||||||
}
|
}
|
||||||
entity.put('district_detail', districtDetail(disCount, cityName));
|
entity.put('district_detail', districtDetail(disCount, cityName));
|
||||||
|
entity.put('district_alias_total_count', districtAliasStat(cityName));
|
||||||
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
|
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
|
||||||
} else {
|
} else {
|
||||||
entity.put('district_detail', [:]);
|
entity.put('district_detail', [:]);
|
||||||
@@ -1032,11 +1037,30 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
return CITY_INFO_CONN.findOne(query);
|
return CITY_INFO_CONN.findOne(query);
|
||||||
}
|
}
|
||||||
|
|
||||||
def districtDetail(def disCount, def cityName) {
|
def districtAliasStat(def cityName){
|
||||||
DBObject query = new BasicDBObject();
|
DBObject query = new BasicDBObject();
|
||||||
query.put('city', cityName);
|
query.put('city', cityName);
|
||||||
query.put('cat', 'district');
|
query.put('cat', 'district');
|
||||||
query.put('visible', 0);
|
query.put('visible', 0);
|
||||||
|
DBObject fields = new BasicDBObject()
|
||||||
|
fields.put("alias", 1)
|
||||||
|
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||||
|
Iterator<DBObject> iter = cur.iterator()
|
||||||
|
def aliasCount = 0
|
||||||
|
while(iter.hasNext()) {
|
||||||
|
def dbo = iter.next()
|
||||||
|
if(dbo && dbo.alias){
|
||||||
|
aliasCount = aliasCount + dbo.alias.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return aliasCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
def districtDetail(def disCount, def cityName) {
|
||||||
|
DBObject query = new BasicDBObject();
|
||||||
|
query.put('city', cityName);
|
||||||
|
query.put('cat', 'district');
|
||||||
|
query.put('status', 0);
|
||||||
|
|
||||||
DBObject fields = new BasicDBObject()
|
DBObject fields = new BasicDBObject()
|
||||||
DISTRICT_FIELDS_MAP.each() {key,skey->
|
DISTRICT_FIELDS_MAP.each() {key,skey->
|
||||||
@@ -1046,7 +1070,6 @@ class CityInfoAnaBatch extends BaseBatch {
|
|||||||
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
|
||||||
Iterator<DBObject> iter = cur.iterator()
|
Iterator<DBObject> iter = cur.iterator()
|
||||||
def countMap = [:]
|
def countMap = [:]
|
||||||
|
|
||||||
while(iter.hasNext()) {
|
while(iter.hasNext()) {
|
||||||
def dbo = iter.next()
|
def dbo = iter.next()
|
||||||
|
|
||||||
|
|||||||
@@ -78,4 +78,16 @@ class FileUtils {
|
|||||||
}
|
}
|
||||||
out.close();
|
out.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def readTxt(def file){
|
||||||
|
def f = new File(file)
|
||||||
|
def ret = [];
|
||||||
|
if(f.exists()){
|
||||||
|
def lines = org.apache.commons.io.FileUtils.readLines(f)
|
||||||
|
for(line in lines) {
|
||||||
|
ret << line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -44,7 +44,7 @@ for(def c : citys){
|
|||||||
def v1 = weekendDetailMap.get(key)
|
def v1 = weekendDetailMap.get(key)
|
||||||
def v2 = lastWeekendDetailMap.get(key)
|
def v2 = lastWeekendDetailMap.get(key)
|
||||||
if(v1 && v2){
|
if(v1 && v2){
|
||||||
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
|
ret << DataHandleUtils.getDoubleRound(4, v1-v2)*100
|
||||||
}else{
|
}else{
|
||||||
ret << "-"
|
ret << "-"
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-13
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
//
|
//
|
||||||
// }
|
// }
|
||||||
// citys = citys[(start+1)..(citys.size()-1)]
|
// citys = citys[(start+1)..(citys.size()-1)]
|
||||||
driver = getLogin()
|
driver = getLogin(driver)
|
||||||
crawlData(driver, citys)
|
crawlData(driver, citys)
|
||||||
|
|
||||||
|
|
||||||
@@ -112,8 +112,8 @@ public class BeiKeCrawlByChrome {
|
|||||||
def s = driver.getPageSource()
|
def s = driver.getPageSource()
|
||||||
def flag = true
|
def flag = true
|
||||||
while(flag){
|
while(flag){
|
||||||
if(s && s.contains("无法访问此网站")){
|
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||||
driver = getLogin()
|
driver = getLogin(driver)
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
Thread.sleep(5000)
|
Thread.sleep(5000)
|
||||||
s = driver.getPageSource()
|
s = driver.getPageSource()
|
||||||
@@ -189,13 +189,9 @@ public class BeiKeCrawlByChrome {
|
|||||||
def detailUrls = []
|
def detailUrls = []
|
||||||
while(!stop){
|
while(!stop){
|
||||||
def url = cityUrl + "pg${page}/"
|
def url = cityUrl + "pg${page}/"
|
||||||
try{
|
driver.get(url)
|
||||||
driver.get(url)
|
// def doc = Jsoup.parse(driver.getPageSource());
|
||||||
}catch(Exception e){
|
def doc = checkDocument(driver, url);
|
||||||
println url +" 出错"
|
|
||||||
}
|
|
||||||
|
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
|
||||||
def elements = JQueryUtils.select(".listContent li", doc)
|
def elements = JQueryUtils.select(".listContent li", doc)
|
||||||
if(elements){
|
if(elements){
|
||||||
for(def e : elements){
|
for(def e : elements){
|
||||||
@@ -426,7 +422,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||||
def list = []
|
def list = []
|
||||||
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
|
def citys = ["福州"]
|
||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = "https:"+e.attr("href")
|
def href = "https:"+e.attr("href")
|
||||||
@@ -438,8 +434,8 @@ public class BeiKeCrawlByChrome {
|
|||||||
return list
|
return list
|
||||||
}
|
}
|
||||||
|
|
||||||
def getLogin(){
|
def getLogin(driver){
|
||||||
WebDriver driver = new ChromeDriver()
|
// WebDriver driver = new ChromeDriver()
|
||||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||||
Thread.sleep(10000)
|
Thread.sleep(10000)
|
||||||
|
|||||||
+25
-12
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
|
|||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.crawl.JQueryUtils
|
import com.fangjia.script.crawl.JQueryUtils
|
||||||
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
@@ -122,10 +123,8 @@ public class LianjiaCrawlByChrome {
|
|||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
def cityMap = getAllCityUrls(driver)
|
def cityMap = getAllCityUrls(driver)
|
||||||
// cityMap = cityMap.reverse()
|
// cityMap = cityMap.reverse()
|
||||||
driver = getLogin()
|
driver = getLogin(driver, "https://sh.lianjia.com/")
|
||||||
crawlData(driver, cityMap)
|
crawlData(driver, cityMap)
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def standardRegion(){
|
def standardRegion(){
|
||||||
@@ -188,8 +187,14 @@ public class LianjiaCrawlByChrome {
|
|||||||
def crawlData(driver, cityList){
|
def crawlData(driver, cityList){
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
|
||||||
|
|
||||||
for(def ret : cityList){
|
for(def ret : cityList){
|
||||||
def city = ret[0]
|
def city = ret[0]
|
||||||
|
if(runedCitys.contains(city)){
|
||||||
|
println city +" 已运行"
|
||||||
|
continue
|
||||||
|
}
|
||||||
def cityUrl = ret[1]
|
def cityUrl = ret[1]
|
||||||
def newUrl = cityUrl+"chengjiao/"
|
def newUrl = cityUrl+"chengjiao/"
|
||||||
driver.get(newUrl)
|
driver.get(newUrl)
|
||||||
@@ -209,6 +214,8 @@ public class LianjiaCrawlByChrome {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||||
|
|
||||||
|
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\链家已抓城市.txt", [], "utf-8", true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,8 +253,8 @@ public class LianjiaCrawlByChrome {
|
|||||||
def s = driver.getPageSource()
|
def s = driver.getPageSource()
|
||||||
def flag = true
|
def flag = true
|
||||||
while(flag){
|
while(flag){
|
||||||
if(s && s.contains("无法访问此网站")){
|
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||||
driver = getLogin()
|
driver = getLogin(driver, url)
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
Thread.sleep(5000)
|
Thread.sleep(5000)
|
||||||
s = driver.getPageSource()
|
s = driver.getPageSource()
|
||||||
@@ -265,7 +272,8 @@ public class LianjiaCrawlByChrome {
|
|||||||
while(!stop){
|
while(!stop){
|
||||||
def url = cityUrl + "pg${page}/"
|
def url = cityUrl + "pg${page}/"
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||||
|
def doc = checkDocument(driver, url);
|
||||||
def elements = JQueryUtils.select(".listContent li", doc)
|
def elements = JQueryUtils.select(".listContent li", doc)
|
||||||
if(elements){
|
if(elements){
|
||||||
for(def e : elements){
|
for(def e : elements){
|
||||||
@@ -495,7 +503,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||||
def cityList = []
|
def cityList = []
|
||||||
def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"]
|
def citys= ["福州"]
|
||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
@@ -507,13 +515,18 @@ public class LianjiaCrawlByChrome {
|
|||||||
return cityList
|
return cityList
|
||||||
}
|
}
|
||||||
|
|
||||||
def getLogin(){
|
def getLogin(driver, url){
|
||||||
WebDriver driver = new ChromeDriver()
|
// WebDriver driver = new ChromeDriver()
|
||||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||||
|
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||||
Thread.sleep(10000)
|
Thread.sleep(10000)
|
||||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
try{
|
||||||
|
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||||
|
}catch(Exception e){
|
||||||
|
Thread.sleep(60000)
|
||||||
|
getLogin(driver, url)
|
||||||
|
}
|
||||||
|
|
||||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user