This commit is contained in:
2023-05-12 18:36:11 +08:00
parent b9780aae68
commit e441a10add
4 changed files with 63 additions and 19 deletions
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -88,7 +88,7 @@ class DistrictFieldAddBatch extends BaseBatch{
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
}
@@ -175,6 +175,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def fields = fieldsMap.keySet() as List
def fieldValueMap = [:]
for(def rhDist : rhRets){
if(rhDist.commid && rhDist.commid!="null"){
continue
}
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
if(!dist){
println c+" "+rhDist.commid+" 不存在"
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
driver = getLogin()
crawlData(driver, citys)
@@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome {
}
}
def checkDocument(driver, url){
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
return Jsoup.parse(s);
}
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
def city = m[0]
def url = m[1]
driver.get(url+"/chengjiao/")
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, newUrl);
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome {
}
def roomStr = arrs[1]
if(roomStr && !roomStr.contains("室")){
roomStr = arrs[2]
}
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
roomStr = roomStr.substring(0, roomStr.indexOf("室"))
@@ -235,16 +255,16 @@ public class BeiKeCrawlByChrome {
}
}
def areaStr = arrs[2].replace("平米", "")
def areaStr = arrs[2]
if(areaStr && !areaStr.contains("平米")){
if(arrs.size()>=4){
areaStr = arrs[3]
}
}
areaStr = areaStr.replace("平米","")
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
if(!detail.area){
areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "")
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
@@ -418,7 +438,8 @@ public class BeiKeCrawlByChrome {
return list
}
def getLogin(driver){
def getLogin(){
WebDriver driver = new ChromeDriver()
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
@@ -438,6 +459,7 @@ public class BeiKeCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(10000)
return driver
}
}
@@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome {
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
getLogin(driver)
driver = getLogin()
crawlData(driver, cityMap)
@@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome {
for(def ret : cityList){
def city = ret[0]
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl)
//def detailUrls = getPageUrls(driver, cityUrl)
driver.get(cityUrl+"chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, newUrl);
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def checkDocument(driver, url){
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
return Jsoup.parse(s);
}
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
@@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome {
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
@@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome {
}
}
def price = e.select(".totalPrice .number").text()
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
}
@@ -490,7 +507,8 @@ public class LianjiaCrawlByChrome {
return cityList
}
def getLogin(driver){
def getLogin(){
WebDriver driver = new ChromeDriver()
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
@@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(20000)
return driver
}
}