+3
-3
@@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
for(def city : citys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
// println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
+2
-2
@@ -38,8 +38,8 @@ class ExportOracleData {
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-02-19")
|
||||
exportBigdataDeal("2022-02-01")
|
||||
exportBigdataForsale("2022-03-19")
|
||||
exportBigdataDeal("2022-03-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+4
-4
@@ -42,17 +42,17 @@ class YouDaDataParser {
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-02-01"
|
||||
static def statDate = "2022-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+32
-12
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="天津"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
}
|
||||
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome {
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!e.text().contains("重庆")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
if(e.text().contains("重庆")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
|
||||
+7
-2
@@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -416,12 +420,13 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["长春","吉林"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+1
-1
@@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
|
||||
def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user