Merge branch 'script_gj' into 'master'

update

See merge request !288
This commit is contained in:
2022-04-20 19:22:40 +08:00
8 changed files with 51 additions and 26 deletions
@@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
// for(def city : citys){
// addDealPriceMatch(city);
// }
for(def city : citys){
addDealPriceMatch(city);
}
// println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-02-19")
exportBigdataDeal("2022-02-01")
exportBigdataForsale("2022-03-19")
exportBigdataDeal("2022-03-01")
//计算结果相关
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -42,17 +42,17 @@ class YouDaDataParser {
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-02-01"
static def statDate = "2022-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
def citys = getAllCityUrls(driver)
citys = citys.reverse()
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
// println i+" "+ m.city
// if(m.city=="天津"){
// start = i
// break;
// }
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
crawlData(driver, cityMap)
crawlData(driver, citys)
}
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
}
}
def crawlData(driver, cityMap){
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome {
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!e.text().contains("重庆")){
cityMap.put(e.text(), "https:"+href)
if(e.text().contains("重庆")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
}
}
return cityMap
return list
}
def getLogin(driver){
@@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -416,12 +420,13 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["长春","吉林"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
// }
}
}
}
return cityMap
@@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"]
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()