@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+4
-1
@@ -88,7 +88,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
if(totalCount){
|
||||
rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
}
|
||||
}
|
||||
@@ -175,6 +175,9 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def fieldValueMap = [:]
|
||||
for(def rhDist : rhRets){
|
||||
if(rhDist.commid && rhDist.commid!="null"){
|
||||
continue
|
||||
}
|
||||
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
|
||||
if(!dist){
|
||||
println c+" "+rhDist.commid+" 不存在"
|
||||
|
||||
+33
-11
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
driver = getLogin()
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
@@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
def city = m[0]
|
||||
def url = m[1]
|
||||
driver.get(url+"/chengjiao/")
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if(roomStr && !roomStr.contains("室")){
|
||||
roomStr = arrs[2]
|
||||
}
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
roomStr = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
@@ -235,16 +255,16 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
def areaStr = arrs[2]
|
||||
if(areaStr && !areaStr.contains("平米")){
|
||||
if(arrs.size()>=4){
|
||||
areaStr = arrs[3]
|
||||
}
|
||||
}
|
||||
areaStr = areaStr.replace("平米","")
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
if(!detail.area){
|
||||
areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "")
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
@@ -418,7 +438,8 @@ public class BeiKeCrawlByChrome {
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
@@ -438,6 +459,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+25
-6
@@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome {
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
getLogin(driver)
|
||||
driver = getLogin()
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
@@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome {
|
||||
for(def ret : cityList){
|
||||
def city = ret[0]
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
driver.get(newUrl)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
driver.get(cityUrl+"chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
}
|
||||
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
@@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def parse(city,e, regionText){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
@@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
@@ -490,7 +507,8 @@ public class LianjiaCrawlByChrome {
|
||||
return cityList
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
@@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user